diff --git a/.drone/drone.yml b/.drone/drone.yml deleted file mode 100644 index 990a142..0000000 --- a/.drone/drone.yml +++ /dev/null @@ -1,144 +0,0 @@ -kind: pipeline -type: exec -name: linux-amd64 - -platform: - arch: amd64 - os: linux - -steps: -- name: test - commands: - - mkdir -p $HOMEPATH/.docker; echo $IMAGE_PULL_SECRETS > $HOMEPATH/.docker/config.json - - make test-style - environment: - VERSION: ${DRONE_TAG:-latest}-linux-amd64 - DEV_REGISTRY: - from_secret: dev_registry - DRYCC_REGISTRY: - from_secret: drycc_registry - CODECOV_TOKEN: - from_secret: codecov_token - IMAGE_PULL_SECRETS: - from_secret: container_pull_secrets - when: - event: - - push - - tag - - pull_request - -- name: publish - commands: - - echo $CONTAINER_PASSWORD | docker login $DRYCC_REGISTRY --username $CONTAINER_USERNAME --password-stdin > /dev/null 2>&1 - - make docker-build docker-immutable-push - environment: - VERSION: ${DRONE_TAG:-latest}-linux-amd64 - DEV_REGISTRY: - from_secret: dev_registry - DRYCC_REGISTRY: - from_secret: drycc_registry - CONTAINER_USERNAME: - from_secret: container_username - CONTAINER_PASSWORD: - from_secret: container_password - when: - event: - - push - - tag - ---- -kind: pipeline -type: exec -name: linux-arm64 - -platform: - arch: arm64 - os: linux - -steps: -- name: publish - commands: - - echo $CONTAINER_PASSWORD | docker login $DRYCC_REGISTRY --username $CONTAINER_USERNAME --password-stdin > /dev/null 2>&1 - - make docker-build docker-immutable-push - environment: - VERSION: ${DRONE_TAG:-latest}-linux-arm64 - DEV_REGISTRY: - from_secret: dev_registry - DRYCC_REGISTRY: - from_secret: drycc_registry - CONTAINER_USERNAME: - from_secret: container_username - CONTAINER_PASSWORD: - from_secret: container_password - when: - event: - - push - - tag - ---- -kind: pipeline -type: docker -name: manifest -image_pull_secrets: -- container_pull_secrets - -steps: - -- name: generate manifest - image: registry.drycc.cc/drycc/python-dev - pull: always - commands: - - sed -i "s/registry.drycc.cc/$${DRYCC_REGISTRY}/g" .drone/manifest.tmpl - environment: - DEV_REGISTRY: - from_secret: dev_registry - DRYCC_REGISTRY: - from_secret: drycc_registry - -- name: publish - image: plugins/manifest - settings: - spec: .drone/manifest.tmpl - username: - from_secret: container_username - password: - from_secret: container_password - environment: - DEV_REGISTRY: - from_secret: dev_registry - DRYCC_REGISTRY: - from_secret: drycc_registry - -trigger: - event: - - push - - tag - -depends_on: -- linux-amd64 -- linux-arm64 - ---- -kind: pipeline -type: exec -name: chart - -steps: -- name: generate chart - commands: - - IMAGE_TAG=$([ ! -z $DRONE_TAG ] && echo \"${DRONE_TAG:1}\" || echo \"canary\") - - sed -i "s/imageTag:\ \"canary\"/imageTag:\ $IMAGE_TAG/g" charts/helmbroker/values.yaml - - helm package -u charts/helmbroker --version $([ -z $DRONE_TAG ] && echo 1.0.0 || echo ${DRONE_TAG#v}) - - echo $CONTAINER_PASSWORD | helm registry login $DRYCC_REGISTRY -u $CONTAINER_USERNAME --password-stdin - - helm push helmbroker-$([ -z $DRONE_TAG ] && echo 1.0.0 || echo ${DRONE_TAG#v}).tgz oci://$DRYCC_REGISTRY/$([ -z $DRONE_TAG ] && echo charts-testing || echo charts) - environment: - DRYCC_REGISTRY: - from_secret: drycc_registry - CONTAINER_USERNAME: - from_secret: container_username - CONTAINER_PASSWORD: - from_secret: container_password - when: - event: - - push - - tag diff --git a/.drone/manifest.tmpl b/.drone/manifest.tmpl deleted file mode 100644 index e482eab..0000000 --- a/.drone/manifest.tmpl +++ /dev/null @@ -1,18 +0,0 @@ -image: registry.drycc.cc/drycc/helmbroker:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}canary{{/if}} -{{#if build.tags}} -tags: -{{#each build.tags}} - - {{this}} -{{/each}} -{{/if}} -manifests: - - - image: registry.drycc.cc/drycc/helmbroker:{{#if build.tag}}{{build.tag}}-{{else}}latest-{{/if}}linux-amd64 - platform: - architecture: amd64 - os: linux - - - image: registry.drycc.cc/drycc/helmbroker:{{#if build.tag}}{{build.tag}}-{{else}}latest-{{/if}}linux-arm64 - platform: - architecture: arm64 - os: linux diff --git a/.gitignore b/.gitignore index d48a71f..54a19de 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,4 @@ vendor/ # generated bintray scripts during ci client/_scripts/ci/bintray-ci.json .idea +.vscode/ \ No newline at end of file diff --git a/.woodpecker/build-linux.yml b/.woodpecker/build-linux.yml new file mode 100644 index 0000000..ef4ecfd --- /dev/null +++ b/.woodpecker/build-linux.yml @@ -0,0 +1,35 @@ +matrix: + platform: + - linux/amd64 + - linux/arm64 + +labels: + type: exec + platform: ${platform} + +steps: +- name: publish-linux + image: bash + commands: + - export VERSION=$([ -z $CI_COMMIT_TAG ] && echo latest || echo $CI_COMMIT_TAG)-$(sed 's#/#-#g' <<< $CI_SYSTEM_PLATFORM) + - echo $CONTAINER_PASSWORD | podman login $DRYCC_REGISTRY --username $CONTAINER_USERNAME --password-stdin > /dev/null 2>&1 + - make podman-build podman-immutable-push + environment: + CODENAME: + from_secret: codename + DEV_REGISTRY: + from_secret: dev_registry + DRYCC_REGISTRY: + from_secret: drycc_registry + CONTAINER_USERNAME: + from_secret: container_username + CONTAINER_PASSWORD: + from_secret: container_password + when: + event: + - push + - tag + - cron + +depends_on: +- test-linux \ No newline at end of file diff --git a/.woodpecker/chart.yaml b/.woodpecker/chart.yaml new file mode 100644 index 0000000..d40739c --- /dev/null +++ b/.woodpecker/chart.yaml @@ -0,0 +1,34 @@ +labels: + type: exec + platform: linux/amd64 + +steps: +- name: generate-chart + image: bash + commands: + - export VERSION=$(sed 's#v##' <<< $CI_COMMIT_TAG) + - export IMAGE_TAG=$([ ! -z $CI_COMMIT_TAG ] && echo \"$VERSION\" || echo \"canary\") + - export APP_VERSION=$([ -z $CI_COMMIT_TAG ] && echo $CI_COMMIT_SHA || echo $VERSION) + - export CHART_VERSION=$([ -z $CI_COMMIT_TAG ] && echo 1.0.0 || echo $VERSION) + - sed -i "s/imageTag:\ \"canary\"/imageTag:\ $IMAGE_TAG/g" charts/$${CI_REPO_NAME}/values.yaml + - sed -i s#{{repository}}#oci://$DRYCC_REGISTRY/$([ -z $CI_COMMIT_TAG ] && echo charts-testing || echo charts)#g charts/$${CI_REPO_NAME}/Chart.yaml + - helm package -u charts/$${CI_REPO_NAME} --version $CHART_VERSION --app-version $APP_VERSION + - echo $CONTAINER_PASSWORD | helm registry login $DRYCC_REGISTRY -u $CONTAINER_USERNAME --password-stdin + - helm push $${CI_REPO_NAME}-$CHART_VERSION.tgz oci://$DRYCC_REGISTRY/$([ -z $CI_COMMIT_TAG ] && echo charts-testing || echo charts) + environment: + DEV_REGISTRY: + from_secret: dev_registry + DRYCC_REGISTRY: + from_secret: drycc_registry + CONTAINER_USERNAME: + from_secret: container_username + CONTAINER_PASSWORD: + from_secret: container_password + when: + event: + - push + - tag + - cron + +depends_on: +- manifest \ No newline at end of file diff --git a/.woodpecker/manifest.tmpl b/.woodpecker/manifest.tmpl new file mode 100644 index 0000000..a5e50ce --- /dev/null +++ b/.woodpecker/manifest.tmpl @@ -0,0 +1,18 @@ +image: registry.drycc.cc/drycc-addons/{{project}}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}canary{{/if}} +{{#if build.tags}} +tags: +{{#each build.tags}} + - {{this}} +{{/each}} +{{/if}} +manifests: + - + image: registry.drycc.cc/drycc-addons/{{project}}:{{#if build.tag}}{{build.tag}}-{{else}}latest-{{/if}}linux-amd64 + platform: + architecture: amd64 + os: linux + - + image: registry.drycc.cc/drycc-addons/{{project}}:{{#if build.tag}}{{build.tag}}-{{else}}latest-{{/if}}linux-arm64 + platform: + architecture: arm64 + os: linux diff --git a/.woodpecker/manifest.yml b/.woodpecker/manifest.yml new file mode 100644 index 0000000..7330ea0 --- /dev/null +++ b/.woodpecker/manifest.yml @@ -0,0 +1,43 @@ +labels: + type: exec + platform: linux/amd64 + +steps: +- name: generate-manifest + image: bash + commands: + - sed -i "s/{{project}}/$${CI_REPO_NAME}/g" .woodpecker/manifest.tmpl + - sed -i "s/registry.drycc.cc/$${DRYCC_REGISTRY}/g" .woodpecker/manifest.tmpl + environment: + DRYCC_REGISTRY: + from_secret: drycc_registry + when: + event: + - tag + - push + - cron + +- name: publish-manifest + image: bash + commands: + - podman run --rm + -e PLUGIN_SPEC=.woodpecker/manifest.tmpl + -e PLUGIN_USERNAME=$CONTAINER_USERNAME + -e PLUGIN_PASSWORD=$CONTAINER_PASSWORD + -e DRONE_TAG=$CI_COMMIT_TAG + -v $(pwd):$(pwd) + -w $(pwd) + docker.io/plugins/manifest + environment: + CONTAINER_USERNAME: + from_secret: container_username + CONTAINER_PASSWORD: + from_secret: container_password + when: + event: + - tag + - push + - cron + +depends_on: +- build-linux diff --git a/.woodpecker/test-linux.yml b/.woodpecker/test-linux.yml new file mode 100644 index 0000000..ab8ee69 --- /dev/null +++ b/.woodpecker/test-linux.yml @@ -0,0 +1,26 @@ +matrix: + platform: + - linux/amd64 + - linux/arm64 + +labels: + type: exec + platform: ${platform} + +steps: +- name: test-linux + image: bash + commands: + - make test upload-coverage + environment: + CODENAME: + from_secret: codename + DEV_REGISTRY: + from_secret: dev_registry + CODECOV_TOKEN: + from_secret: codecov_token + when: + event: + - push + - tag + - cron diff --git a/Makefile b/Makefile index 148a8c0..fed80c8 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,13 @@ # If DRYCC_REGISTRY is not set, try to populate it from legacy DEV_REGISTRY DRYCC_REGISTRY ?= $(DEV_REGISTRY) -IMAGE_PREFIX ?= drycc +IMAGE_PREFIX ?= drycc-addons COMPONENT ?= helmbroker SHORT_NAME ?= $(COMPONENT) PLATFORM ?= linux/amd64,linux/arm64 include versioning.mk -SHELLCHECK_PREFIX := docker run --rm -v ${CURDIR}:/workdir -w /workdir ${DEV_REGISTRY}/drycc/go-dev shellcheck +SHELLCHECK_PREFIX := podman run --rm -v ${CURDIR}:/workdir -w /workdir ${DEV_REGISTRY}/drycc/go-dev shellcheck SHELL_SCRIPTS = $(wildcard rootfs/bin/*) $(shell find "rootfs" -name '*.sh') $(wildcard _scripts/*.sh) # Test processes used in quick unit testing @@ -19,42 +19,39 @@ check-kubectl: exit 2; \ fi -check-docker: - @if [ -z $$(which docker) ]; then \ - echo "Missing \`docker\` client which is required for development"; \ +check-podman: + @if [ -z $$(which podman) ]; then \ + echo "Missing \`podman\` client which is required for development"; \ exit 2; \ fi -build: docker-build +build: podman-build -docker-build: check-docker - docker build ${DOCKER_BUILD_FLAGS} -t ${IMAGE} rootfs - docker tag ${IMAGE} ${MUTABLE_IMAGE} +podman-build: check-podman + podman build ${PODMAN_BUILD_FLAGS} --build-arg CODENAME=${CODENAME} -t ${IMAGE} rootfs + podman tag ${IMAGE} ${MUTABLE_IMAGE} -docker-buildx: check-docker - docker buildx build --platform ${PLATFORM} -t ${IMAGE} rootfs --push +podman-build-test: check-podman + podman build ${PODMAN_BUILD_FLAGS} --build-arg CODENAME=${CODENAME} -t ${IMAGE}.test -f rootfs/Dockerfile.test rootfs -docker-build-test: check-docker - docker build ${DOCKER_BUILD_FLAGS} -t ${IMAGE}.test -f rootfs/Dockerfile.test rootfs - -deploy: check-kubectl docker-build docker-push +deploy: check-kubectl podman-build podman-push kubectl --namespace=drycc patch deployment drycc-$(COMPONENT) --type='json' -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/image", "value":"$(IMAGE)"}]' -clean: check-docker - docker rmi $(IMAGE) +clean: check-podman + podman rmi $(IMAGE) -full-clean: check-docker - docker images -q $(IMAGE_PREFIX)/$(COMPONENT) | xargs docker rmi -f +full-clean: check-podman + podman images -q $(IMAGE_PREFIX)/$(COMPONENT) | xargs podman rmi -f test: test-style test-unit test-functional -test-style: docker-build-test +test-style: podman-build-test $(shell chown -R 1001:1001 ${CURDIR}) - docker run --rm -v ${CURDIR}:/tmp/test -w /tmp/test/rootfs ${IMAGE}.test /tmp/test/rootfs/bin/test-style + podman run --rm -v ${CURDIR}:/tmp/test -w /tmp/test/rootfs ${IMAGE}.test /tmp/test/rootfs/bin/test-style ${SHELLCHECK_PREFIX} $(SHELL_SCRIPTS) -test-unit: docker-build-test - @echo "Implement in the future" +test-unit: podman-build-test + podman run --rm -v ${CURDIR}:/tmp/test -w /tmp/test/rootfs ${IMAGE}.test /tmp/test/rootfs/bin/test-unit test-functional: @echo "Implement functional tests in _tests directory" @@ -64,6 +61,6 @@ test-integration: upload-coverage: $(eval CI_ENV := $(shell curl -s https://codecov.io/env | bash)) - docker run --rm ${CI_ENV} -v ${CURDIR}:/tmp/test -w /tmp/test/rootfs ${IMAGE}.test /tmp/test/rootfs/bin/upload-coverage + podman run --rm ${CI_ENV} -v ${CURDIR}:/tmp/test -w /tmp/test/rootfs -e CODECOV_TOKEN=${CODECOV_TOKEN} ${IMAGE}.test /tmp/test/rootfs/bin/upload-coverage -.PHONY: check-kubectl check-docker build docker-build docker-build-test deploy clean commit-hook full-clean test test-style test-unit test-functional test-integration upload-coverage +.PHONY: check-kubectl check-podman build podman-build podman-build-test deploy clean commit-hook full-clean test test-style test-unit test-functional test-integration upload-coverage diff --git a/README.md b/README.md index 676028a..3ffb184 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Drycc Helmbroker -[![Build Status](https://drone.drycc.cc/api/badges/drycc/helmbroker/status.svg)](https://drone.drycc.cc/drycc/helmbroker) -[![codecov.io](https://codecov.io/github/drycc/helmbroker/coverage.svg?branch=main)](https://codecov.io/github/drycc/helmbroker?branch=main) +[![Build Status](https://woodpecker.drycc.cc/api/badges/drycc-addons/helmbroker/status.svg)](https://woodpecker.drycc.cc/drycc-addons/helmbroker) +[![codecov.io](https://codecov.io/github/drycc-addons/helmbroker/coverage.svg?branch=main)](https://codecov.io/github/drycc-addons/helmbroker?branch=main) Drycc (pronounced DAY-iss) Workflow is an open source Platform as a Service (PaaS) that adds a developer-friendly layer to any [Kubernetes](http://kubernetes.io) cluster, making it easy to deploy and manage applications on your own servers. diff --git a/charts/helmbroker/Chart.yaml b/charts/helmbroker/Chart.yaml index 1fc3fc4..6e8d2aa 100644 --- a/charts/helmbroker/Chart.yaml +++ b/charts/helmbroker/Chart.yaml @@ -1,10 +1,15 @@ name: helmbroker home: https://github.com/drycc/helmbroker apiVersion: v2 +appVersion: 1.0.0 dependencies: - name: common repository: oci://registry.drycc.cc/charts - version: ~1.1.1 + version: ~1.1.2 + - name: valkey + repository: {{repository}} + version: x.x.x + condition: valkey.enabled description: Drycc Workflow helmbroker. maintainers: - name: Drycc Team diff --git a/charts/helmbroker/templates/_helpers.tpl b/charts/helmbroker/templates/_helpers.tpl index 8a3e2dc..3e9e40f 100644 --- a/charts/helmbroker/templates/_helpers.tpl +++ b/charts/helmbroker/templates/_helpers.tpl @@ -3,31 +3,31 @@ env: - name: "TZ" value: {{ .Values.time_zone | default "UTC" | quote }} -- name: USERNAME +- name: HELMBROKER_USERNAME value: {{ if .Values.username | default "" | ne "" }}{{ .Values.username }}{{ else }}{{ randAlphaNum 32 }}{{ end }} -- name: PASSWORD +- name: HELMBROKER_PASSWORD value: {{ if .Values.password | default "" | ne "" }}{{ .Values.password }}{{ else }}{{ randAlphaNum 32 }}{{ end }} +{{- if (.Values.valkeyUrl) }} +- name: HELMBROKER_VALKEY_URL + valueFrom: + secretKeyRef: + name: helmbroker-creds + key: valkey-url +{{- else if .Values.valkey.enabled }} +- name: VALKEY_PASSWORD + valueFrom: + secretKeyRef: + name: valkey-creds + key: password +- name: HELMBROKER_VALKEY_URL + value: "redis://:$(VALKEY_PASSWORD)@drycc-valkey:26379/0?master_set=drycc" +{{- end }} {{- range $key, $value := .Values.environment }} - name: {{ $key }} value: {{ $value | quote }} {{- end }} {{- end }} - -{{/* Generate helmbroker deployment limits */}} -{{- define "helmbroker.limits" -}} -{{- if or (.Values.limits_cpu) (.Values.limits_memory) }} -resources: - limits: -{{- if (.Values.limits_cpu) }} - cpu: {{.Values.limits_cpu}} -{{- end }} -{{- if (.Values.limits_memory) }} - memory: {{.Values.limits_memory}} -{{- end }} -{{- end }} -{{- end }} - {{/* Generate helmbroker deployment volumeMounts */}} {{- define "helmbroker.volumeMounts" }} volumeMounts: diff --git a/charts/helmbroker/templates/helmbroker-celery-deployment.yaml b/charts/helmbroker/templates/helmbroker-celery-deployment.yaml new file mode 100644 index 0000000..1b39b78 --- /dev/null +++ b/charts/helmbroker/templates/helmbroker-celery-deployment.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: drycc-helmbroker-celery + labels: + heritage: drycc + annotations: + component.drycc.cc/version: {{ .Values.imageTag }} +spec: + replicas: {{ .Values.celery.replicas }} + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + selector: + matchLabels: + app: drycc-helmbroker-celery + template: + metadata: + labels: {{- include "common.labels.standard" . | nindent 8 }} + app: drycc-helmbroker-celery + spec: + affinity: + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.celery.podAffinityPreset.type "component" "" "extraMatchLabels" .Values.celery.podAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.celery.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.celery.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.celery.nodeAffinityPreset.type "key" .Values.celery.nodeAffinityPreset.key "values" .Values.celery.nodeAffinityPreset.values ) | nindent 10 }} + serviceAccount: drycc-helmbroker + initContainers: + - name: drycc-helmbroker-celery-init + image: registry.drycc.cc/drycc/python-dev:latest + imagePullPolicy: {{.Values.imagePullPolicy}} + args: + - netcat + - -v + - -u + - http://drycc-helmbroker + {{- include "helmbroker.envs" . | indent 10 }} + containers: + {{- range $key := (list "low" "middle" "high") }} + - name: drycc-helmbroker-celery-{{$key}} + image: {{$.Values.imageRegistry}}/{{$.Values.imageOrg}}/helmbroker:{{$.Values.imageTag}} + imagePullPolicy: {{$.Values.imagePullPolicy}} + {{- if $.Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" $.Values.diagnosticMode.command "context" $) | nindent 8 }} + args: {{- include "common.tplvalues.render" (dict "value" $.Values.diagnosticMode.args "context" $) | nindent 8 }} + {{- else }} + args: + - /bin/bash + - -c + - celery --app helmbroker worker -n {{uuidv4}}@%h --queues helmbroker.{{$key}} --autoscale=32,1 --loglevel=WARNING + {{- end }} + {{- with index $.Values "celery" "resources" }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- include "helmbroker.envs" $ | indent 8 }} + {{- include "helmbroker.volumeMounts" $ | indent 8 }} + {{- end }} + {{- include "helmbroker.volumes" . | indent 6 }} + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsUser: 1001 diff --git a/charts/helmbroker/templates/helmbroker-celery.yaml b/charts/helmbroker/templates/helmbroker-celery.yaml deleted file mode 100644 index 9443b4b..0000000 --- a/charts/helmbroker/templates/helmbroker-celery.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: drycc-helmbroker-celery - labels: - heritage: drycc - annotations: - component.drycc.cc/version: {{ .Values.imageTag }} -spec: - replicas: {{ .Values.celeryReplicas }} - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - type: RollingUpdate - selector: - matchLabels: - app: drycc-helmbroker-celery - template: - metadata: - labels: - app: drycc-helmbroker-celery - spec: - affinity: - podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.celery.podAffinityPreset.type "key" .Values.celery.podAffinityPreset.key "values" .Values.celery.podAffinityPreset.values ) | nindent 10 }} - podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.celery.podAntiAffinityPreset.type "key" .Values.celery.podAntiAffinityPreset.key "values" .Values.celery.podAntiAffinityPreset.values ) | nindent 10 }} - nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.celery.nodeAffinityPreset.type "key" .Values.celery.nodeAffinityPreset.key "values" .Values.celery.nodeAffinityPreset.values ) | nindent 10 }} - serviceAccount: drycc-helmbroker - containers: - - name: drycc-helmbroker-celery - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/helmbroker:{{.Values.imageTag}} - imagePullPolicy: {{.Values.imagePullPolicy}} - args: - - /bin/bash - - -c - - celery -A helmbroker worker --autoscale=32,1 --loglevel=info - {{- include "helmbroker.limits" . | indent 10 }} - {{- include "helmbroker.envs" . | indent 10 }} - {{- include "helmbroker.volumeMounts" . | indent 10 }} - {{- include "helmbroker.volumes" . | indent 6 }} diff --git a/charts/helmbroker/templates/helmbroker-certificate.yaml b/charts/helmbroker/templates/helmbroker-certificate.yaml deleted file mode 100644 index 1410a48..0000000 --- a/charts/helmbroker/templates/helmbroker-certificate.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if .Values.certManagerEnabled }} -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: drycc-helmbroker -spec: - secretName: drycc-helmbroker-certificate-auto - issuerRef: - name: drycc-cluster-issuer - kind: ClusterIssuer - dnsNames: - - drycc-helmbroker.{{ .Values.platformDomain }} - privateKey: - rotationPolicy: Always -{{- end }} diff --git a/charts/helmbroker/templates/helmbroker-clusterrolebinding.yaml b/charts/helmbroker/templates/helmbroker-clusterrolebinding.yaml index f193a90..ddf07d3 100644 --- a/charts/helmbroker/templates/helmbroker-clusterrolebinding.yaml +++ b/charts/helmbroker/templates/helmbroker-clusterrolebinding.yaml @@ -1,7 +1,7 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: drycc:drycc-helmbroker + name: {{ printf "%s:drycc-helmbroker" .Release.Namespace | quote }} labels: app: drycc-helmbroker heritage: drycc diff --git a/charts/helmbroker/templates/helmbroker-cm.yaml b/charts/helmbroker/templates/helmbroker-cm.yaml index b9a13ac..d13e2ee 100644 --- a/charts/helmbroker/templates/helmbroker-cm.yaml +++ b/charts/helmbroker/templates/helmbroker-cm.yaml @@ -13,4 +13,8 @@ data: - name: {{ .name }} url: {{ .url }} {{- end }} + {{- if .Values.addonValues }} + addon-values: | + {{- (tpl .Values.addonValues $) | nindent 4 }} + {{- end }} {{- end }} diff --git a/charts/helmbroker/templates/helmbroker-cronjob-daily.yaml b/charts/helmbroker/templates/helmbroker-cronjob-daily.yaml index c608862..825e3c8 100644 --- a/charts/helmbroker/templates/helmbroker-cronjob-daily.yaml +++ b/charts/helmbroker/templates/helmbroker-cronjob-daily.yaml @@ -21,10 +21,16 @@ spec: - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/helmbroker:{{.Values.imageTag}} imagePullPolicy: {{.Values.imagePullPolicy}} name: drycc-helmbroker-cleaner + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }} + {{- else }} args: - /bin/bash - -c - python -m helmbroker.cleaner + python -m helmbroker.database.fetch + {{- end }} {{- include "helmbroker.envs" . | indent 12 }} {{- include "helmbroker.volumeMounts" . | indent 12 }} {{- include "helmbroker.volumes" . | indent 10 }} \ No newline at end of file diff --git a/charts/helmbroker/templates/helmbroker-deployment.yaml b/charts/helmbroker/templates/helmbroker-deployment.yaml index 459142d..45cc4cb 100644 --- a/charts/helmbroker/templates/helmbroker-deployment.yaml +++ b/charts/helmbroker/templates/helmbroker-deployment.yaml @@ -7,7 +7,7 @@ metadata: annotations: component.drycc.cc/version: {{ .Values.imageTag }} spec: - replicas: {{ .Values.replicas }} + replicas: {{ .Values.api.replicas }} strategy: rollingUpdate: maxSurge: 1 @@ -18,45 +18,67 @@ spec: app: drycc-helmbroker template: metadata: - labels: + labels: {{- include "common.labels.standard" . | nindent 8 }} app: drycc-helmbroker spec: affinity: - podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.api.podAffinityPreset.type "key" .Values.api.podAffinityPreset.key "values" .Values.api.podAffinityPreset.values ) | nindent 10 }} - podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.api.podAntiAffinityPreset.type "key" .Values.api.podAntiAffinityPreset.key "values" .Values.api.podAntiAffinityPreset.values ) | nindent 10 }} + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.api.podAffinityPreset.type "component" "" "extraMatchLabels" .Values.api.podAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.api.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.api.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }} nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.api.nodeAffinityPreset.type "key" .Values.api.nodeAffinityPreset.key "values" .Values.api.nodeAffinityPreset.values ) | nindent 10 }} serviceAccount: drycc-helmbroker initContainers: - - name: loader + - name: drycc-helmbroker-init + image: registry.drycc.cc/drycc/python-dev:latest + imagePullPolicy: {{.Values.imagePullPolicy}} + args: + - netcat + - -v + - -u + - $(HELMBROKER_VALKEY_URL) + {{- include "helmbroker.envs" . | indent 10 }} + - name: drycc-helmbroker-fetch image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/helmbroker:{{.Values.imageTag}} imagePullPolicy: {{.Values.imagePullPolicy}} args: - /bin/bash - -c - - python -m helmbroker.loader + - python -m helmbroker.database.fetch {{- include "helmbroker.envs" . | indent 10 }} {{- include "helmbroker.volumeMounts" . | indent 10 }} containers: - - name: drycc-helmbroker - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/helmbroker:{{.Values.imageTag}} - imagePullPolicy: {{.Values.imagePullPolicy}} - livenessProbe: - httpGet: - path: /healthz - port: 8000 - initialDelaySeconds: 30 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: /readiness - port: 8000 - initialDelaySeconds: 30 - timeoutSeconds: 10 - periodSeconds: 5 - ports: - - containerPort: 8000 - name: http - {{- include "helmbroker.limits" . | indent 10 }} - {{- include "helmbroker.envs" . | indent 10 }} - {{- include "helmbroker.volumeMounts" . | indent 10 }} + - name: drycc-helmbroker + image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/helmbroker:{{.Values.imageTag}} + imagePullPolicy: {{.Values.imagePullPolicy}} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 8 }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 8 }} + {{- end }} + {{- if not .Values.diagnosticMode.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: 8000 + initialDelaySeconds: 30 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: /readiness + port: 8000 + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 5 + {{- end }} + ports: + - containerPort: 8000 + name: http + {{- with index .Values "api" "resources" }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- include "helmbroker.envs" . | indent 8 }} + {{- include "helmbroker.volumeMounts" . | indent 8 }} {{- include "helmbroker.volumes" . | indent 6 }} + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsUser: 1001 diff --git a/charts/helmbroker/templates/helmbroker-ingress.yaml b/charts/helmbroker/templates/helmbroker-ingress.yaml deleted file mode 100644 index 61f8040..0000000 --- a/charts/helmbroker/templates/helmbroker-ingress.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: "helmbroker-api-server" - labels: - app: "helmbroker" - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - annotations: - kubernetes.io/tls-acme: "true" -spec: - {{- if not (eq .Values.ingressClass "") }} - ingressClassName: "{{ .Values.ingressClass }}" - {{ end }} - rules: - - host: drycc-helmbroker.{{ .Values.platformDomain }} - http: - paths: - - pathType: Prefix - {{- if eq .Values.ingressClass "gce" "alb" }} - path: /* - {{- else }}{{/* Has annotations but ingress class is not "gce" nor "alb" */}} - path: / - {{- end }} - backend: - service: - name: drycc-helmbroker - port: - number: 80 - {{- if .Values.certManagerEnabled }} - tls: - - secretName: drycc-helmbroker-certificate-auto - hosts: - - drycc-helmbroker.{{ .Values.platformDomain }} - {{- end }} diff --git a/charts/helmbroker/templates/helmbroker-secret-creds.yaml b/charts/helmbroker/templates/helmbroker-secret-creds.yaml new file mode 100644 index 0000000..29b1bc3 --- /dev/null +++ b/charts/helmbroker/templates/helmbroker-secret-creds.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: helmbroker-creds + labels: + heritage: drycc +data: + {{- if (.Values.valkeyUrl) }} + valkey-url: {{ .Values.valkeyUrl | b64enc }} + {{- end }} \ No newline at end of file diff --git a/charts/helmbroker/values.yaml b/charts/helmbroker/values.yaml index 1dd0c29..65b9f54 100644 --- a/charts/helmbroker/values.yaml +++ b/charts/helmbroker/values.yaml @@ -1,61 +1,92 @@ -imageOrg: "drycc" +imageOrg: "drycc-addons" imageTag: "canary" imageRegistry: "registry.drycc.cc" imagePullPolicy: "Always" -replicas: 1 -# limitsCpu: "100m" -# limitsMemory: "50Mi" + + +## Enable diagnostic mode +## +diagnosticMode: + ## @param diagnosticMode.enabled Enable diagnostic mode (all probes will be disabled and the command will be overridden) + ## + enabled: false + ## @param diagnosticMode.command Command to override all containers + ## + command: + - sleep + ## @param diagnosticMode.args Args to override all containers + ## + args: + - infinity ## config the helm-broker repositories repositories: -- name: drycc-helm-broker - url: https://github.com/drycc/addons/releases/download/latest/index.yaml - -celeryReplicas: 1 + - name: drycc-helm-broker + url: https://github.com/drycc/addons/releases/download/latest/index.yaml # broker_credentials: # Optional Usernames and passwords that will be required to communicate with service broker username: admin password: admin +# Configuring this will no longer use the built-in valkey component +valkeyUrl: "" + +# Any custom controller environment variables +# can be specified as key-value pairs under environment +# this is usually a non required setting. environment: - RESERVED_NAMES: "drycc, drycc-helmbroker" - # HELMBROKER_CELERY_BROKER: "" - # HELMBROKER_CELERY_BACKEND: "" + # HELMBROKER_DEBUG: true + # HELMBROKER_CONFIG_ROOT: /etc/helmbroker api: + replicas: 1 + resources: {} + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi nodeAffinityPreset: key: "drycc.cc/node" type: "soft" values: - - "true" + - "true" podAffinityPreset: - key: "security" type: "" - values: - - "drycc-security" + extraMatchLabels: + security: "drycc-security" podAntiAffinityPreset: - key: "app" type: "soft" - values: - - "drycc-helmbroker" + extraMatchLabels: + app: "drycc-helmbroker" celery: + replicas: 1 + resources: {} + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi nodeAffinityPreset: key: "drycc.cc/node" type: "soft" values: - - "true" + - "true" podAffinityPreset: - key: "security" type: "" - values: - - "drycc-security" + extraMatchLabels: + security: "drycc-security" podAntiAffinityPreset: - key: "app" type: "soft" - values: - - "drycc-helmbroker-celery" + extraMatchLabels: + app: "drycc-helmbroker-celery" + +# Default override of addon values +addonValues: {} persistence: enabled: true @@ -64,15 +95,5 @@ persistence: storageClass: "" volumeName: "" -ingressClass: "" -# A domain name consists of one or more parts. -# Periods (.) are used to separate these parts. -# Each part must be 1 to 63 characters in length and can contain lowercase letters, digits, and hyphens (-). -# It must start and end with a lowercase letter or digit. -clusterDomain: "cluster.local" -# The public resolvable hostname to build your cluster with. -# -# This will be the hostname that is used to build endpoints such as "drycc-helmbroker.$HOSTNAME" -platformDomain: "" -# Whether cert_manager is enabled to automatically generate helmbroker certificates -certManagerEnabled: true \ No newline at end of file +valkey: + enabled: true diff --git a/codecov.yml b/codecov.yml index c81ef32..b83b98f 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,4 +1,4 @@ -# documentation is at https://codecov.io/gh/drycc/helmbroker/settings/yaml +# documentation is at https://codecov.io/gh/drycc-addons/helmbroker/settings/yaml comment: layout: header, diff coverage: diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 5b5baa3..85d3134 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -1,11 +1,12 @@ -FROM registry.drycc.cc/drycc/base:bullseye +ARG CODENAME +FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/workspace \ - PYTHON_VERSION="3.10.6" \ - HELM_VERSION="3.9.4" \ - KUBECTL_VERSION="1.25.0" + PYTHON_VERSION="3.14" \ + HELM_VERSION="4.1.3" \ + KUBECTL_VERSION="1.35.3" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} @@ -14,7 +15,7 @@ COPY . ${DRYCC_HOME_DIR} WORKDIR ${DRYCC_HOME_DIR} RUN buildDeps='musl-dev' \ - && install-packages ${buildDeps} openssl ca-certificates \ + && install-packages ${buildDeps} patch openssl ca-certificates \ && install-stack python ${PYTHON_VERSION} \ && install-stack helm ${HELM_VERSION} \ && install-stack kubectl ${KUBECTL_VERSION} && . init-stack \ diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test index 78edf19..fe48d53 100644 --- a/rootfs/Dockerfile.test +++ b/rootfs/Dockerfile.test @@ -1,11 +1,12 @@ -FROM registry.drycc.cc/drycc/base:bullseye +ARG CODENAME +FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/workspace \ - PYTHON_VERSION="3.10.6" \ - HELM_VERSION="3.9.4" \ - KUBECTL_VERSION="1.25.0" + PYTHON_VERSION="3.14" \ + HELM_VERSION="4.1.3" \ + KUBECTL_VERSION="1.35.3" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} @@ -14,7 +15,9 @@ COPY . ${DRYCC_HOME_DIR} WORKDIR ${DRYCC_HOME_DIR} RUN buildDeps='musl-dev'; \ - install-packages ${buildDeps} openssl ca-certificates \ + install-packages ${buildDeps} git openssl ca-certificates \ + && curl -SsL https://cli.codecov.io/latest/$([ $(dpkg --print-architecture) == "arm64" ] && echo linux-arm64 || echo linux)/codecov -o /usr/local/bin/codecov \ + && chmod +x /usr/local/bin/codecov \ && install-stack python ${PYTHON_VERSION} \ && install-stack helm ${HELM_VERSION} \ && install-stack kubectl ${KUBECTL_VERSION} && . init-stack \ diff --git a/rootfs/bin/test-unit b/rootfs/bin/test-unit new file mode 100755 index 0000000..1d1d54e --- /dev/null +++ b/rootfs/bin/test-unit @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# +# This script is designed to be run inside the container +# + +coverage run -m unittest tests/test_*.py diff --git a/rootfs/bin/upload-coverage b/rootfs/bin/upload-coverage index 53433f3..492452b 100755 --- a/rootfs/bin/upload-coverage +++ b/rootfs/bin/upload-coverage @@ -6,4 +6,8 @@ # fail hard and fast even on pipelines set -eou pipefail -codecov --required +coverage report -m > coverage.txt + +if [[ -n $CODECOV_TOKEN ]]; then + codecov upload-process --plugin=noop -t "$CODECOV_TOKEN" +fi diff --git a/rootfs/dev_requirements.txt b/rootfs/dev_requirements.txt index f8bc513..ca19f67 100644 --- a/rootfs/dev_requirements.txt +++ b/rootfs/dev_requirements.txt @@ -1,16 +1,10 @@ # test module # test # Run "make test-unit" for the % of code exercised during tests -coverage==5.3 +coverage==7.6.1 # Run "make test-style" to check python syntax and style -flake8==3.8.3 - -# code coverage report at https://codecov.io/github/drycc/controller -codecov==2.1.9 +flake8==7.1.1 # mock out python-requests, mostly k8s -requests-mock==1.8.0 - -# tail a log and pipe into tbgrep to find all tracebacks -tbgrep==0.3.0 +requests-mock==1.12.1 diff --git a/rootfs/helmbroker/broker.py b/rootfs/helmbroker/broker.py index a629b52..c920789 100644 --- a/rootfs/helmbroker/broker.py +++ b/rootfs/helmbroker/broker.py @@ -1,6 +1,6 @@ import os +import logging import time -import shutil from typing import Union, List, Optional from openbrokerapi.catalog import ServicePlan @@ -13,11 +13,17 @@ UpdateDetails, UpdateServiceSpec, DeprovisionDetails, \ DeprovisionServiceSpec, LastOperation, OperationState -from .utils import get_instance_path, get_chart_path, get_plan_path, \ - get_addon_path, get_addon_updateable, get_addon_bindable, InstanceLock, \ - load_instance_meta, load_binding_meta, dump_instance_meta, \ - load_addons_meta -from .tasks import provision, bind, deprovision, update +from .utils import verify_parameters, new_instance_lock, verify_parameters_by_plan +from .database.fetch import fetch_chart_plan +from .database.query import get_instance_path, get_chart_path, get_plan_path, \ + get_addon_updateable, get_addon_bindable, get_addon_allow_params, \ + get_addon_archive, get_binding_file, get_instance_file +from .database.metadata import load_instance_meta, load_binding_meta, load_addons_meta, \ + save_instance_meta +from .tasks import provision, bind, deprovision, update, unbind +from .config import INSTANCE_NAME_LENS + +logger = logging.getLogger(__name__) class HelmServiceBroker(ServiceBroker): @@ -38,36 +44,35 @@ def provision(self, details: ProvisionDetails, async_allowed: bool, **kwargs) -> ProvisionedServiceSpec: + logger.debug(f"*** provision instance {instance_id}") + # verify instance_name length + if len(details.context["instance_name"]) > INSTANCE_NAME_LENS: + raise ErrBadRequest( + msg=f"The length of the instance name cannot exceed {INSTANCE_NAME_LENS}.") instance_path = get_instance_path(instance_id) if os.path.exists(instance_path): raise ErrInstanceAlreadyExists() if not async_allowed: raise ErrAsyncRequired() + if get_addon_archive(details.service_id): + raise ErrBadRequest( + msg="This addon has archived.") + allow_params = get_addon_allow_params(details.service_id) + not_allow_keys, required_keys = verify_parameters( + allow_params, details.parameters) + if not_allow_keys: + raise ErrBadRequest( + msg="parameters %s does not allowed" % not_allow_keys) + if required_keys: + raise ErrBadRequest( + msg="required parameters %s not exists" % required_keys) os.makedirs(instance_path, exist_ok=True) - chart_path, plan_path = ( - get_chart_path(instance_id), get_plan_path(instance_id)) - addon_chart_path, addon_plan_path = ( - get_addon_path(details.service_id, details.plan_id)) - shutil.copytree(addon_chart_path, chart_path) - shutil.copytree(addon_plan_path, plan_path) - data = { - "id": instance_id, - "details": { - "service_id": details.service_id, - "plan_id": details.plan_id, - "context": details.context, - "parameters": details.parameters, - }, - "last_operation": { - "state": OperationState.IN_PROGRESS.value, - "operation": "provision", - "description": ( - "provision %s in progress at %s" % ( - instance_id, time.time())) - } - } - with InstanceLock(instance_id): - dump_instance_meta(instance_id, data) + chart_path, plan_path = get_chart_path(instance_id), get_plan_path(instance_id) + fetch_chart_plan(details.service_id, chart_path, details.plan_id, plan_path) + # verify instance-schema + msg = verify_parameters_by_plan(instance_id, details.parameters) + if msg: + raise ErrBadRequest(msg) provision.delay(instance_id, details) return ProvisionedServiceSpec(state=ProvisionState.IS_ASYNC) @@ -88,6 +93,7 @@ def bind(self, async_allowed: bool, **kwargs ) -> Binding: + logger.debug(f"*** bind instance {instance_id}") is_addon_bindable = get_addon_bindable(details.service_id) if not is_addon_bindable: raise ErrBadRequest( @@ -100,9 +106,6 @@ def bind(self, instance_path = get_instance_path(instance_id) if os.path.exists(f'{instance_path}/bind.json'): raise ErrBindingAlreadyExists() - chart_path, plan_path = ( - get_chart_path(instance_id), get_plan_path(instance_id)) - shutil.copy(f'{plan_path}/bind.yaml', f'{chart_path}/templates') bind(instance_id, binding_id, details, async_allowed, **kwargs) data = load_binding_meta(instance_id) if data["last_operation"]["state"] == OperationState.SUCCEEDED.value: @@ -118,11 +121,9 @@ def unbind(self, async_allowed: bool, **kwargs ) -> UnbindSpec: - instance_path = get_instance_path(instance_id) - binding_info = f'{instance_path}/binding.json' - if os.path.exists(binding_info): - os.remove(binding_info) - return UnbindSpec(is_async=False) + logger.debug(f"*** unbind instance {instance_id}") + unbind.delay(instance_id) + return UnbindSpec(is_async=True) def update(self, instance_id: str, @@ -130,6 +131,7 @@ def update(self, async_allowed: bool, **kwargs ) -> UpdateServiceSpec: + logger.debug(f"*** update instance {instance_id}") instance_path = get_instance_path(instance_id) if not os.path.exists(instance_path): raise ErrBadRequest(msg="Instance %s does not exist" % instance_id) @@ -137,16 +139,31 @@ def update(self, if not is_plan_updateable: raise ErrBadRequest( msg="Instance %s does not updateable" % instance_id) + allow_params = get_addon_allow_params(details.service_id) + logger.debug( + f"service instance update parameters: {details.parameters}") + not_allow_keys, required_keys = verify_parameters( + allow_params, details.parameters) + if not_allow_keys: + raise ErrBadRequest( + msg="parameters %s does not allowed" % not_allow_keys) + if required_keys: + raise ErrBadRequest( + msg="required parameters %s not exists" % required_keys) if not async_allowed: raise ErrAsyncRequired() if details.plan_id is not None: - plan_path = get_plan_path(instance_id) - # delete the pre plan - shutil.rmtree(plan_path, ignore_errors=True) - _, addon_plan_path = get_addon_path( - details.service_id, details.plan_id) - # add the new plan - shutil.copytree(addon_plan_path, plan_path) + chart_path, plan_path = get_chart_path(instance_id), get_plan_path(instance_id) + fetch_chart_plan(details.service_id, chart_path, details.plan_id, plan_path) + # verify instance-schema + msg = verify_parameters_by_plan(instance_id, details.parameters) + if msg: + raise ErrBadRequest(msg) + data = load_instance_meta(instance_id) + data['last_operation']["state"] = OperationState.IN_PROGRESS.value + data['last_operation']["description"] = ( + f"update {instance_id} in progress at {time.time()}") + save_instance_meta(instance_id, data) update.delay(instance_id, details) return UpdateServiceSpec(is_async=True) @@ -155,9 +172,10 @@ def deprovision(self, details: DeprovisionDetails, async_allowed: bool, **kwargs) -> DeprovisionServiceSpec: + logger.debug(f"*** deprovision instance {instance_id}") if not os.path.exists(get_instance_path(instance_id)): raise ErrInstanceDoesNotExist() - with InstanceLock(instance_id): + with new_instance_lock(instance_id): data = load_instance_meta(instance_id) operation = data["last_operation"]["operation"] if operation == "provision": @@ -174,11 +192,14 @@ def last_operation(self, operation_data: Optional[str], **kwargs ) -> LastOperation: - data = load_instance_meta(instance_id) - return LastOperation( - OperationState(data["last_operation"]["state"]), - data["last_operation"]["description"] - ) + logger.debug(f"*** last_operation instance {instance_id}") + if os.path.exists(get_instance_file(instance_id)): + data = load_instance_meta(instance_id) + return LastOperation( + OperationState(data["last_operation"]["state"]), + data["last_operation"]["description"] + ) + return LastOperation(OperationState.IN_PROGRESS) def last_binding_operation(self, instance_id: str, @@ -186,8 +207,11 @@ def last_binding_operation(self, operation_data: Optional[str], **kwargs ) -> LastOperation: - data = load_binding_meta(instance_id) - return LastOperation( - OperationState(data["last_operation"]["state"]), - data["last_operation"]["description"] - ) + logger.debug(f"*** last_binding_operation instance {instance_id}") + if os.path.exists(get_binding_file(instance_id)): + data = load_binding_meta(instance_id) + return LastOperation( + OperationState(data["last_operation"]["state"]), + data["last_operation"]["description"] + ) + return LastOperation(OperationState.IN_PROGRESS) diff --git a/rootfs/helmbroker/celery.py b/rootfs/helmbroker/celery.py index 5ecf2f4..45d4181 100644 --- a/rootfs/helmbroker/celery.py +++ b/rootfs/helmbroker/celery.py @@ -1,33 +1,91 @@ import os +from urllib.parse import urlparse, parse_qs, urlencode +from kombu import Exchange, Queue from celery import Celery +from .config import VALKEY_URL -class Config: +class Config(object): # Celery Configuration Options - timezone = "Asia/Shanghai" enable_utc = True task_serializer = 'pickle' accept_content = frozenset([ - 'application/data', - 'application/text', - 'application/json', - 'application/x-python-serialize', + 'application/data', + 'application/text', + 'application/json', + 'application/x-python-serialize', ]) task_track_started = True task_time_limit = 30 * 60 worker_max_tasks_per_child = 200 + worker_prefetch_multiplier = 1 result_expires = 24 * 60 * 60 + cache_backend = 'django-cache' + task_default_queue = 'helmbroker.middle' + task_default_exchange = 'helmbroker.priority' + task_default_routing_key = 'helmbroker.priority.middle' + broker_transport_options = {"queue_order_strategy": "sorted"} + task_create_missing_queues = True + task_inherit_parent_priority = True + broker_connection_retry_on_startup = True worker_cancel_long_running_tasks_on_connection_loss = True -app = Celery( - 'helmbroker', - broker=os.environ.get("HELMBROKER_CELERY_BROKER"), - backend=os.environ.get("HELMBROKER_CELERY_BACKEND"), - include=['helmbroker.tasks'] +app = Celery('helmbroker') +app.config_from_object(Config()) +app.conf.update( + timezone=os.environ.get('TZ', 'UTC'), + task_routes={ + 'helmbroker.tasks.provision': { + 'queue': 'helmbroker.high', + 'exchange': 'helmbroker.priority', 'routing_key': 'helmbroker.priority.high', + }, + 'helmbroker.tasks.update': { + 'queue': 'helmbroker.high', + 'exchange': 'helmbroker.priority', 'routing_key': 'helmbroker.priority.high', + }, + 'helmbroker.tasks.bind': { + 'queue': 'helmbroker.high', + 'exchange': 'helmbroker.priority', 'routing_key': 'helmbroker.priority.high', + }, + 'helmbroker.tasks.deprovision': { + 'queue': 'helmbroker.middle', + 'exchange': 'helmbroker.priority', 'routing_key': 'helmbroker.priority.middle', + }, + }, + task_queues=( + Queue( + 'helmbroker.low', exchange=Exchange('helmbroker.priority', type="direct"), + routing_key='helmbroker.priority.low', + ), + Queue( + 'helmbroker.high', exchange=Exchange('helmbroker.priority', type="direct"), + routing_key='helmbroker.priority.high', + ), + Queue( + 'helmbroker.middle', exchange=Exchange('helmbroker.priority', type="direct"), + routing_key='helmbroker.priority.middle', + ), + ), +) +app.autodiscover_tasks(("helmbroker.tasks",)) +url = urlparse(VALKEY_URL) +query = parse_qs(url.query) +broker_transport_options = {"queue_order_strategy": "sorted", "visibility_timeout": 43200} +result_backend_transport_options = {} +if 'master_set' in query: + master_name = query.pop('master_set')[0] + password = url.netloc.split("@")[0].split(":")[1] + kwargs = {'sentinel_kwargs': {'password': password}, 'master_name': master_name} + broker_transport_options.update(kwargs) + result_backend_transport_options.update(kwargs) + VALKEY_URL = f"sentinel://{url.netloc}{url.path}?{urlencode(query)}" +app.conf.update( + broker_url=VALKEY_URL, + result_backend=VALKEY_URL, + broker_transport_options=broker_transport_options, + result_backend_transport_options=result_backend_transport_options, ) - -app.config_from_object(Config) if __name__ == '__main__': app.start() diff --git a/rootfs/helmbroker/cleaner.py b/rootfs/helmbroker/cleaner.py index 0b01814..6cd456a 100644 --- a/rootfs/helmbroker/cleaner.py +++ b/rootfs/helmbroker/cleaner.py @@ -6,8 +6,8 @@ from openbrokerapi.service_broker import OperationState from .config import INSTANCES_PATH -from .tasks import deprovision -from .utils import get_instance_file, load_instance_meta +from .database.query import get_instance_file +from .database.metadata import load_instance_meta logger = logging.getLogger(__name__) @@ -21,10 +21,6 @@ def clean_instance(): interval = time.time() - data["last_modified_time"] state = data["last_operation"]["state"] operation = data["last_operation"]["operation"] - if interval > 3600 * 24 and ( - operation == "deprovision" - and state != OperationState.SUCCEEDED): - deprovision.delay(instance_id) if operation == "deprovision": if state == OperationState.SUCCEEDED or ( interval > 3600 * 24 diff --git a/rootfs/helmbroker/config.py b/rootfs/helmbroker/config.py index 682d58a..c83ad94 100644 --- a/rootfs/helmbroker/config.py +++ b/rootfs/helmbroker/config.py @@ -1,14 +1,18 @@ import os -HELMBROKER_ROOT = os.environ.get("HELMBROKER_ROOT", '/etc/helmbroker') -ADDONS_PATH = os.path.join(HELMBROKER_ROOT, 'addons') -CONFIG_PATH = os.path.join(HELMBROKER_ROOT, 'config') -INSTANCES_PATH = os.path.join(HELMBROKER_ROOT, 'instances') +CONFIG_ROOT = os.environ.get("HELMBROKER_CONFIG_ROOT", '/etc/helmbroker') +ADDONS_PATH = os.path.join(CONFIG_ROOT, 'addons') +CONFIG_PATH = os.path.join(CONFIG_ROOT, 'config') +INSTANCES_PATH = os.path.join(CONFIG_ROOT, 'instances') -USERNAME = os.environ.get('USERNAME') -PASSWORD = os.environ.get('PASSWORD') + +USERNAME = os.environ.get('HELMBROKER_USERNAME') +PASSWORD = os.environ.get('HELMBROKER_PASSWORD') + +VALKEY_URL = os.environ.get("HELMBROKER_VALKEY_URL", 'redis://localhost:6379/0') +INSTANCE_NAME_LENS = int(os.environ.get("INSTANCE_NAME_LENS", '32')) class Config: - DEBUG = bool(os.environ.get('DRYCC_DEBUG', True)) + DEBUG = bool(os.environ.get('HELMBROKER_DEBUG', False)) diff --git a/rootfs/helmbroker/database/__init__.py b/rootfs/helmbroker/database/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rootfs/helmbroker/database/fetch.py b/rootfs/helmbroker/database/fetch.py new file mode 100644 index 0000000..d2d8102 --- /dev/null +++ b/rootfs/helmbroker/database/fetch.py @@ -0,0 +1,100 @@ +import os +import tarfile +import requests +import yaml +import json +import glob +import shutil +import tempfile +import collections + + +def fetch_addons(repository): + if not repository: + return + temp = tempfile.TemporaryDirectory() + try: + index_name = repository['url'].split('/')[-1] + # download index.yaml + remote_index = requests.get(repository['url']).content.decode(encoding="utf-8") + # new index + with open(f"{temp.name}/{index_name}", 'w') as f: + f.write(remote_index) + remote_index = yaml.load(remote_index, Loader=yaml.Loader) + # save index.yaml addons + download_urls = {} + for addon_name, addon_metas in remote_index.get('entries', {}).items(): + download_urls[addon_name] = {} + for addon_meta in addon_metas: + url = "/".join(repository["url"].split("/")[0:-1]) + meta_name = f'{addon_name}-{addon_meta["version"]}' + addon_tgz_url = f'{url}/{meta_name}.tgz' + _fetch_addon(addon_tgz_url, f'{temp.name}/{meta_name}') + return _read_addons_meta(temp.name) + finally: + temp.cleanup() + + +def fetch_chart_plan(addon_id, chart_path, plan_id, plan_path): + from .query import get_addon_meta + addon_meta = get_addon_meta(addon_id) + addon_plan = [plan for plan in addon_meta['plans'] if plan['id'] == plan_id][0] + temp = tempfile.TemporaryDirectory() + try: + _fetch_addon(addon_meta['url'], temp.name) + shutil.rmtree(chart_path, ignore_errors=True) + shutil.rmtree(plan_path, ignore_errors=True) + shutil.copytree(os.path.join(temp.name, "chart", addon_meta["name"]), chart_path) + shutil.copytree(os.path.join(temp.name, "plans", addon_plan['name']), plan_path) + finally: + temp.cleanup() + + +def _fetch_addon(url, dest): + with tempfile.TemporaryFile(suffix=".tgz") as tgz_file: + tgz_file.write(requests.get(url).content) + tgz_file.flush() + tgz_file.seek(0) + os.makedirs(dest, exist_ok=True) + with tarfile.open(fileobj=tgz_file, mode="r:gz") as tarobj: + for tarinfo in tarobj: + tarobj.extract(tarinfo.name, dest, filter='data') + filename1 = os.path.join(dest, "meta.yaml") + with open(filename1, "r") as f1: + meta = yaml.load(stream=f1, Loader=yaml.Loader) + meta['url'] = url + meta['version'] = str(meta['version']) + meta['tags'] = [tag.strip() for tag in meta.get('tags').split(',') if tag.strip()] + with open(os.path.join(dest, "meta.json"), "w") as f2: + json.dump(meta, f2) + os.remove(filename1) + + +def _read_addons_meta(addons_path): + addons_meta = collections.OrderedDict() + for metafile in glob.glob(os.path.join(addons_path, "*", "meta.json")): + with open(metafile) as f1: + meta = json.load(f1) + meta['plans'] = [] + metapath = os.path.join(os.path.dirname(metafile)) + for planfile in glob.glob(os.path.join(metapath, "plans", "*", "meta.yaml")): + with open(planfile, 'r') as f2: + plan = yaml.load(f2.read(), Loader=yaml.Loader) + meta["plans"].append(plan) + addons_meta[meta['displayName']] = meta + return addons_meta + + +def main(): + from ..config import CONFIG_PATH + from .metadata import save_addons_meta + addons_meta = collections.OrderedDict() + with open(f'{CONFIG_PATH}/repositories', 'r') as f: + repositories = yaml.load(f.read(), Loader=yaml.Loader) + for repository in repositories: + addons_meta.update(fetch_addons(repository)) + save_addons_meta(addons_meta) + + +if __name__ == '__main__': + main() diff --git a/rootfs/helmbroker/database/metadata.py b/rootfs/helmbroker/database/metadata.py new file mode 100644 index 0000000..2877aaa --- /dev/null +++ b/rootfs/helmbroker/database/metadata.py @@ -0,0 +1,184 @@ +import os +import json +import time +import logging +import jsonschema + +from ..utils import get_valkey_client +from ..config import ADDONS_PATH + +logger = logging.getLogger(__name__) + +INSTANCE_META_SCHEMA = { + "type": "object", + "properties": { + "id": {"type": "string"}, + "details": { + "type": "object", + "properties": { + "service_id": {"type": "string"}, + "plan_id": {"type": "string"}, + "context": {"type": "object"}, + "parameters": { + 'oneOf': [{'type': 'object'}, {'type': 'null'}] + }, + }, + "required": [ + "service_id", "plan_id", "context" + ] + }, + "last_operation": { + "type": "object", + "properties": { + "state": {"type": "string"}, + "operation": {"type": "string"}, + "description": {"type": "string"} + } + }, + "last_modified_time": {"type": "number"} + }, +} + +BINDING_META_SCHEMA = { + "type": "object", + "properties": { + "id": {"type": "string"}, + "credentials": { + "type": "object", + }, + "last_operation": { + "type": "object", + "properties": { + "state": {"type": "string"}, + "description": {"type": "string"} + } + }, + "last_modified_time": {"type": "number"} + } +} + +ADDONS_META_SCHEMA = { + "type": "object", + "patternProperties": { + ".*": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "version": {"type": "string"}, + "bindable": {"type": "boolean"}, + "instances_retrievable": {"type": "boolean"}, + "bindings_retrievable": {"type": "boolean"}, + "allow_context_updates": {"type": "boolean"}, + "description": {"type": "string"}, + "tags": {"type": "array", "items": {"type": "string"}}, + "requires": {"type": "array"}, + "metadata": {"type": "object"}, + "plan_updateable": {"type": "boolean"}, + "dashboard_client": {"type": "object"}, + "plans": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "description": {"type": "string"}, + "metadata": {"type": "object"}, + "free": {"type": "boolean"}, + "bindable": {"type": "boolean"}, + "binding_rotatable": {"type": "boolean"}, + "plan_updateable": {"type": "boolean"}, + "schemas": {"type": "object"}, + "maximum_polling_duration": {"type": "integer"}, + "maintenance_info": {"type": "object"}, + }, + "required": ["id", "name", "description"] + }, + "minItems": 1, + + }, + }, + "required": ["id", "name", "description", "bindable", "version", "plans"] + }, + }, + "additionalProperties": False, +} + + +def save_instance_meta(instance_id, data): + cache_key = f"helmbroker:instance:{instance_id}" + from .query import get_instance_file + data["last_modified_time"] = time.time() + file = get_instance_file(instance_id) + jsonschema.validate(instance=data, schema=INSTANCE_META_SCHEMA) + + json_data = json.dumps(data, sort_keys=True, indent=2) + with open(file, "w") as f: + f.write(json_data) + get_valkey_client().set(cache_key, json_data) + + +def load_instance_meta(instance_id): + cache_key = f"helmbroker:instance:{instance_id}" + valkey = get_valkey_client() + + json_data = valkey.get(cache_key) + if not json_data: + from .query import get_instance_file + file = get_instance_file(instance_id) + with open(file) as f: + json_data = f.read() + valkey.set(cache_key, json_data) + return json.loads(json_data) + + +def save_binding_meta(instance_id, data): + from .query import get_binding_file + cache_key = f"helmbroker:binding:{instance_id}" + data["last_modified_time"] = time.time() + file = get_binding_file(instance_id) + jsonschema.validate(instance=data, schema=BINDING_META_SCHEMA) + + json_data = json.dumps(data, sort_keys=True, indent=2) + with open(file, "w") as f: + f.write(json_data) + get_valkey_client().set(cache_key, json_data) + + +def load_binding_meta(instance_id): + from .query import get_binding_file + cache_key = f"helmbroker:binding:{instance_id}" + valkey = get_valkey_client() + json_data = valkey.get(cache_key) + if not json_data: + file = get_binding_file(instance_id) + with open(file, 'r') as f: + json_data = f.read() + valkey.set(cache_key, json_data) + return json.loads(json_data) + + +def save_addons_meta(data): + cache_key = "helmbroker:addons" + os.makedirs(ADDONS_PATH, exist_ok=True) + file = os.path.join(ADDONS_PATH, "addons.json") + jsonschema.validate(instance=data, schema=ADDONS_META_SCHEMA) + + json_data = json.dumps(data, sort_keys=True, indent=2) + with open(file, "w") as f: + f.write(json_data) + get_valkey_client().set(cache_key, json_data) + + +def load_addons_meta(): + cache_key = "helmbroker:addons" + valkey = get_valkey_client() + + json_data = valkey.get(cache_key) + if not json_data: + file = os.path.join(ADDONS_PATH, "addons.json") + with open(file, 'r') as f: + json_data = f.read() + valkey.set(cache_key, json_data) + return json.loads(json_data) diff --git a/rootfs/helmbroker/database/query.py b/rootfs/helmbroker/database/query.py new file mode 100644 index 0000000..741cfc3 --- /dev/null +++ b/rootfs/helmbroker/database/query.py @@ -0,0 +1,116 @@ +import os +import base64 + +from ..utils import command +from ..config import INSTANCES_PATH +from .metadata import load_addons_meta + + +def get_instance_path(instance_id): + return os.path.join(INSTANCES_PATH, instance_id) + + +def get_instance_file(instance_id): + return os.path.join(get_instance_path(instance_id), "instance.json") + + +def get_chart_path(instance_id): + return os.path.join(get_instance_path(instance_id), "chart") + + +def get_plan_path(instance_id): + return os.path.join(get_instance_path(instance_id), "plan") + + +def get_plan_schema_path(instance_id): + return os.path.join(get_instance_path(instance_id), "plan", "instance-schema.json") + + +def get_hooks_path(instance_id): + return os.path.join(get_plan_path(instance_id), "hooks") + + +def get_hooks_result_file(instance_id): + return os.path.join(get_instance_path(instance_id), "hooks-result.json") + + +def get_binding_file(instance_id): + return os.path.join(get_instance_path(instance_id), "binding.json") + + +def get_backups_path(instance_id): + return os.path.join(get_instance_path(instance_id), "backups") + + +def get_addon_values_file(instance_id): + return os.path.join(get_instance_path(instance_id), "addon-values.yaml") + + +def get_custom_addon_values_file(instance_id): + return os.path.join(get_instance_path(instance_id), "custom-addon-values.yaml") + + +def get_addon_updateable(addon_id): + addon_meta = get_addon_meta(addon_id) + return addon_meta.get('plan_updateable', False) + + +def get_addon_bindable(addon_id): + addon_meta = get_addon_meta(addon_id) + return addon_meta.get('bindable', False) + + +def get_addon_allow_params(addon_id): + addon_meta = get_addon_meta(addon_id) + return addon_meta.get('allow_parameters', []) + + +def get_addon_archive(addon_id): + addon_meta = get_addon_meta(addon_id) + return addon_meta.get('archive', False) + + +def get_cred_value(ns, source): + if source.get('serviceRef'): + return _get_service_key_value(ns, source['serviceRef']) + if source.get('configMapRef'): + return _get_config_map_key_value(ns, source['configMapRef']) + if source.get('secretKeyRef'): + return _get_secret_key_value(ns, source['secretKeyRef']) + return -1, 'invalid valueFrom' + + +def get_addon_meta(addon_id): + addons_meta = load_addons_meta() + addons_meta = [ + addon for addon in [addons for _, addons in addons_meta.items()] + if addon['id'] == addon_id + ] + return addons_meta[0] if len(addons_meta) > 0 else None + + +def _get_service_key_value(ns, service_ref): + args = [ + "get", "svc", service_ref['name'], "-n", ns, + '-o', f"jsonpath=\'{service_ref['jsonpath']}\'", + ] + return command("kubectl", *args) + + +def _get_config_map_key_value(ns, config_map_ref): + args = [ + "get", "cm", config_map_ref['name'], "-n", ns, + '-o', f"jsonpath=\'{config_map_ref['jsonpath']}\'", + ] + return command("kubectl", *args) + + +def _get_secret_key_value(ns, secret_ref): + args = [ + "get", "secret", secret_ref['name'], "-n", ns, '-o', + f"jsonpath=\'{secret_ref['jsonpath']}\'", + ] + status, output = command("kubectl", *args) + if status == 0: + output = base64.b64decode(output).decode() + return status, output diff --git a/rootfs/helmbroker/database/savepoint.py b/rootfs/helmbroker/database/savepoint.py new file mode 100644 index 0000000..ba97960 --- /dev/null +++ b/rootfs/helmbroker/database/savepoint.py @@ -0,0 +1,65 @@ +import os +import json +import logging +import yaml +import shutil +import datetime + +from ..config import CONFIG_PATH +from .query import get_instance_path, get_backups_path, get_addon_meta, get_addon_values_file, \ + get_custom_addon_values_file, get_hooks_result_file + +logger = logging.getLogger(__name__) + + +def backup_instance(instance_id): + now = datetime.datetime.now(datetime.timezone.utc) + backup_path = os.path.join(get_backups_path(instance_id), now.isoformat()) + os.makedirs(backup_path, exist_ok=True) + + hooks_result_file = get_hooks_result_file(instance_id) + if os.path.exists(hooks_result_file): + shutil.copy(hooks_result_file, backup_path) + addon_values_file = get_addon_values_file(instance_id) + if os.path.exists(addon_values_file): + shutil.copy(addon_values_file, backup_path) + custom_addon_values_file = get_custom_addon_values_file(instance_id) + if os.path.exists(custom_addon_values_file): + shutil.copy(custom_addon_values_file, backup_path) + + instance_path = get_instance_path(instance_id) + shutil.copytree(os.path.join(instance_path, "plan"), os.path.join(backup_path, "plan")) + shutil.copytree(os.path.join(instance_path, "chart"), os.path.join(backup_path, "chart")) + + +def save_raw_values(instance_id, data): + file = get_custom_addon_values_file(instance_id) + with open(file, "w") as f: + f.write(data) + return file + + +def save_addon_values(service_id, instance_id): + file = get_addon_values_file(instance_id) + service = get_addon_meta(service_id) + logger.debug(f"save_addon_values service: {service}") + if not os.path.exists(f'{CONFIG_PATH}/addon-values'): + return None + with open(file, "w") as fw: + with open(f'{CONFIG_PATH}/addon-values', 'r') as f: + addons_values = yaml.load(f.read(), Loader=yaml.Loader) + logger.debug(f"save_addon_values addons_values: {addons_values}") + addon_values = addons_values.get(service["name"], {}).\ + get(service["version"], {}) + logger.debug(f"save_addon_values addon_values: {addon_values}") + if not addon_values: + return None + fw.write(yaml.dump(addon_values)) + return file + + +def save_hooks_result(instance_id, data): + file = get_hooks_result_file(instance_id) + with open(file, "w") as f: + f.write(json.dumps(data, sort_keys=True, indent=2)) + return file diff --git a/rootfs/helmbroker/gunicorn/config.py b/rootfs/helmbroker/gunicorn/config.py index 4038c74..716f9e5 100644 --- a/rootfs/helmbroker/gunicorn/config.py +++ b/rootfs/helmbroker/gunicorn/config.py @@ -18,14 +18,14 @@ def worker_int(worker): - """Print a stack trace when a worker receives a SIGINT or SIGQUIT signal.""" # noqa + """Print a stack trace when a worker receives a SIGINT or SIGQUIT signal.""" worker.log.warning('worker terminated') import traceback traceback.print_stack() def worker_abort(worker): - """Print a stack trace when a worker receives a SIGABRT signal, generally on timeout.""" # noqa + """Print a stack trace when a worker receives a SIGABRT signal, generally on timeout.""" worker.log.warning('worker aborted') import traceback traceback.print_stack() diff --git a/rootfs/helmbroker/gunicorn/logging.py b/rootfs/helmbroker/gunicorn/logging.py index f9e65f3..41a3f9e 100644 --- a/rootfs/helmbroker/gunicorn/logging.py +++ b/rootfs/helmbroker/gunicorn/logging.py @@ -1,12 +1,12 @@ -import os from gunicorn.glogging import Logger +from helmbroker.config import Config class Logging(Logger): def access(self, resp, req, environ, request_time): # health check endpoints are only logged in debug mode if ( - not os.environ.get('DEBUG', False) and + not Config.DEBUG and req.path in ['/readiness', '/healthz'] ): return diff --git a/rootfs/helmbroker/loader.py b/rootfs/helmbroker/loader.py deleted file mode 100644 index 9afa699..0000000 --- a/rootfs/helmbroker/loader.py +++ /dev/null @@ -1,115 +0,0 @@ -import os -import shutil -import tarfile -import requests -import yaml - -from .config import ADDONS_PATH, CONFIG_PATH -from .utils import dump_addons_meta - - -def download_file(url, dest): - if not os.path.exists(dest): - os.system(f'mkdir -p {dest}') - filename = url.split('/')[-1] - file = requests.get(url) - with open(f"{dest}/{filename}", 'wb') as f: - f.write(file.content) - if filename.endswith(".yaml") or filename.endswith(".yml"): - return yaml.load(file.content.decode(encoding="utf-8"), - Loader=yaml.Loader) - - -def read_file(filename): - if not os.path.exists(filename): - return - with open(filename, 'r') as f: - file_content = f.read() - return file_content - - -def save_file(content, dest, filename): - if not os.path.exists(dest): - os.system(f'mkdir -p {dest}') - with open(f"{dest}/{filename}", 'w') as f: - f.write(content) - - -def extract_tgz(tgz_file, dest): - if not os.path.exists(tgz_file): - return - if not os.path.exists(dest): - os.system(f'mkdir -p {dest}') - - tarobj = tarfile.open(tgz_file, "r:gz") - for tarinfo in tarobj: - tarobj.extract(tarinfo.name, dest) - tarobj.close() - - -def addons_meta_file(): - meta_files = [] - # get meta.yaml - for root, dirnames, filenames in os.walk(ADDONS_PATH): - for filename in filenames: - if filename == 'meta.yaml': - meta_files.append(os.path.join(root, filename)) - meta_files = [meta_file.split(ADDONS_PATH)[1] for meta_file in meta_files] - addons_meta = [] - plans_meta = [] - for meta_file in meta_files: - if len(meta_file.split('/')) == 3: - addons_meta.append(meta_file.split('/')[1:]) - else: - plans_meta.append(meta_file.split('/')[1:]) - addons_dict = {} - for addon_meta in addons_meta: - with open(f'{ADDONS_PATH}/{"/".join(addon_meta)}', 'r') as f: - meta = yaml.load(f.read(), Loader=yaml.Loader) - meta['tags'] = meta.get('tags').split(', ') if meta.get('tags') else [] # noqa - meta['plans'] = [] - addons_dict[meta['name']] = meta - - for plan_meta in plans_meta: - with open(f'{ADDONS_PATH}/{"/".join(plan_meta)}', 'r') as f: - addons_mata = yaml.load(f.read(), Loader=yaml.Loader) - addons_dict[f'{"-".join(plan_meta[0].split("-")[0:-1])}']['plans'].append(addons_mata) # noqa - dump_addons_meta(addons_dict) - - -def load_addons(repository): - if not repository: - return - index_name = repository['url'].split('/')[-1] - local_index_file = f'{ADDONS_PATH}/{index_name}' - # download index.yaml - remote_index = requests.get(repository['url']).content.decode( - encoding="utf-8") - # compare index.yaml, is update - local_index = read_file(local_index_file) - if local_index and remote_index == local_index: - return - # delete old repository catalog - if os.path.exists(ADDONS_PATH): - shutil.rmtree(ADDONS_PATH, ignore_errors=True) - else: - os.makedirs(ADDONS_PATH, exist_ok=True) - # new index - save_file(remote_index, ADDONS_PATH, index_name) - remote_index = yaml.load(remote_index, Loader=yaml.Loader) - # save index.yaml addons - for k, v in remote_index.get('entries', {}).items(): - for _ in v: - url = "/".join(repository["url"].split("/")[0:-1]) - tgz_name = f'{_["name"]}-{_["version"]}' - addon_tgz_url = f'{url}/{tgz_name}.tgz' - download_file(addon_tgz_url, ADDONS_PATH) - extract_tgz(f'{ADDONS_PATH}/{tgz_name}.tgz', - f'{ADDONS_PATH}/{tgz_name}') - addons_meta_file() - - -if __name__ == '__main__': - with open(f'{CONFIG_PATH}/repositories', 'r') as f: - repositories = yaml.load(f.read(), Loader=yaml.Loader) - load_addons(repositories[0]) diff --git a/rootfs/helmbroker/tasks.py b/rootfs/helmbroker/tasks.py index 3370b65..69f2f30 100644 --- a/rootfs/helmbroker/tasks.py +++ b/rootfs/helmbroker/tasks.py @@ -1,106 +1,134 @@ import os import time -import shutil import yaml +import logging +import shutil from openbrokerapi.service_broker import ProvisionDetails, OperationState, \ UpdateDetails, BindDetails from .celery import app -from .utils import get_plan_path, get_chart_path, get_cred_value, \ - InstanceLock, dump_instance_meta, dump_binding_meta, load_instance_meta, \ - get_instance_file, helm +from .utils import helm, format_params_to_helm_args, new_instance_lock, run_instance_hooks + +from .database.metadata import save_instance_meta, save_binding_meta, load_instance_meta, \ + load_binding_meta +from .database.savepoint import save_addon_values, backup_instance +from .database.query import get_plan_path, get_chart_path, get_cred_value, get_binding_file + +logger = logging.getLogger(__name__) @app.task(serializer='pickle') def provision(instance_id: str, details: ProvisionDetails): - with InstanceLock(instance_id): + logger.debug(f"*** task provision instance: {instance_id}, before lock") + with ( + new_instance_lock(instance_id), + run_instance_hooks(instance_id, "provision") as (status, output) + ): + logger.debug(f"*** task provision instance: {instance_id}") + backup_instance(instance_id) + # create instance.json + data = { + "id": instance_id, "last_operation": {}, + "details": { + "service_id": details.service_id, "plan_id": details.plan_id, + "context": details.context, + "parameters": details.parameters if details.parameters else {}, + }, + } + if status != 0: + data["last_operation"]["state"] = OperationState.FAILED.value + data["last_operation"]["description"] = f"provision {instance_id} error: {output}" + save_instance_meta(instance_id, data) + return + data["last_operation"]["state"] = OperationState.IN_PROGRESS.value + data["last_operation"]["operation"] = "provision" + data["last_operation"]["description"] = ( + f"provision {instance_id} in progress at {time.time()}") + save_instance_meta(instance_id, data) chart_path = get_chart_path(instance_id) bind_yaml = f'{chart_path}/templates/bind.yaml' if os.path.exists(bind_yaml): os.remove(bind_yaml) - if os.path.exists(f'{chart_path}/requirements.lock'): - args = [ - "dependency", - "update", - chart_path, - ] + if os.path.exists(f'{chart_path}/Chart.yaml'): + args = ["dependency", "update", chart_path] helm(instance_id, *args) values_file = os.path.join(get_plan_path(instance_id), "values.yaml") args = [ - "install", - details.context["instance_name"], - chart_path, - "--namespace", - details.context["namespace"], - "--create-namespace", - "--wait", - "--timeout", - "25m0s", - "-f", - values_file, - "--set", - f"fullnameOverride=helmbroker-{details.context['instance_name']}" + "install", details.context["instance_name"], chart_path, + "--namespace", details.context["namespace"], "--create-namespace", + "--wait", "--timeout", "25m0s", "-f", values_file, + "--set", f"fullnameOverride=helmbroker-{details.context['instance_name']}" ] - + addon_values_file = save_addon_values(details.service_id, instance_id) + if addon_values_file: + args.insert(9, "-f") + args.insert(10, addon_values_file) + logger.debug(f"helm install parameters: {details.parameters}") + args = format_params_to_helm_args(instance_id, details.parameters, args) + logger.debug(f"helm install args: {args}") status, output = helm(instance_id, *args) - data = load_instance_meta(instance_id) if status != 0: data["last_operation"]["state"] = OperationState.FAILED.value - data["last_operation"]["description"] = ( - "provision error:\n%s" % output) + data["last_operation"]["description"] = f"provision {instance_id} error: {output}" else: data["last_operation"]["state"] = OperationState.SUCCEEDED.value data["last_operation"]["description"] = ( - "provision succeeded at %s" % time.time()) - dump_instance_meta(instance_id, data) + f"provision {instance_id} succeeded at {time.time()}") + save_instance_meta(instance_id, data) @app.task(serializer='pickle') def update(instance_id: str, details: UpdateDetails): - data = load_instance_meta(instance_id) - if details.service_id: - data['details']['service_id'] = details.service_id - if details.plan_id: - data['details']['service_id'] = details.plan_id - if details.context: - data['details']['context'] = details.context - if details.parameters: - data['details']['service_id'] = details.parameters - data['last_operation'] = { - "state": OperationState.IN_PROGRESS.value, - "description": ( - "update %s in progress at %s" % (instance_id, time.time())) - } - dump_instance_meta(instance_id, data) - chart_path = get_chart_path(instance_id) - values_file = os.path.join(get_plan_path(instance_id), "values.yaml") - args = [ - "upgrade", - details.context["instance_name"], - chart_path, - "--namespace", - details.context["namespace"], - "--create-namespace", - "--wait", - "--timeout", - "25m0s", - "-f", - values_file, - "--set", - f"fullnameOverride=helmbroker-{details.context['instance_name']}" - ] - - status, output = helm(instance_id, *args) - if status != 0: - data["last_operation"]["state"] = OperationState.FAILED.value - data["last_operation"]["description"] = ( - "update %s failed: %s" % (instance_id, output)) - else: - data["last_operation"]["state"] = OperationState.SUCCEEDED.value - data["last_operation"]["description"] = ( - "update %s succeeded at %s" % (instance_id, time.time())) - dump_instance_meta(instance_id, data) + logger.debug(f"*** task update instance: {instance_id}, before lock") + with ( + new_instance_lock(instance_id), + run_instance_hooks(instance_id, "update") as (status, output) + ): + logger.debug(f"*** task update instance: {instance_id}") + backup_instance(instance_id) + data = load_instance_meta(instance_id) + if details.service_id: + data['details']['service_id'] = details.service_id + if details.plan_id: + data['details']['service_id'] = details.plan_id + if details.context: + data['details']['context'] = details.context + if details.parameters: + params = data['details']['parameters'] + params.update(details.parameters) + # remove the key which value is null + data['details']['parameters'] = {k: v for k, v in params.items() if v != ""} + if status != 0: + data["last_operation"]["state"] = OperationState.FAILED.value + data["last_operation"]["description"] = f"update {instance_id} failed: {output}" + save_instance_meta(instance_id, data) + return + chart_path = get_chart_path(instance_id) + values_file = os.path.join(get_plan_path(instance_id), "values.yaml") + args = [ + "upgrade", details.context["instance_name"], chart_path, + "--namespace", details.context["namespace"], "--create-namespace", + "--wait", "--timeout", "25m0s", "--reset-then-reuse-values", "-f", values_file, + "--set", f"fullnameOverride=helmbroker-{details.context['instance_name']}" + ] + addon_values_file = save_addon_values(details.service_id, instance_id) + if addon_values_file: + args.insert(10, "-f") + args.insert(11, addon_values_file) + params = data['details']['parameters'] + logger.debug(f"helm upgrade parameters: {params}") + args = format_params_to_helm_args(instance_id, params, args) + logger.debug(f"helm upgrade args: {args}") + status, output = helm(instance_id, *args) + if status != 0: + data["last_operation"]["state"] = OperationState.FAILED.value + data["last_operation"]["description"] = f"update {time.time()} failed: {output}" + else: + data["last_operation"]["state"] = OperationState.SUCCEEDED.value + data["last_operation"]["description"] = ( + f"update {instance_id} succeeded at {time.time()}") + save_instance_meta(instance_id, data) @app.task(serializer='pickle') @@ -109,90 +137,127 @@ def bind(instance_id: str, details: BindDetails, async_allowed: bool, **kwargs): - data = { - "binding_id": binding_id, - "credentials": {}, - "last_operation": { - "state": OperationState.IN_PROGRESS.value, - "description": ( - "binding %s in progress at %s" % (binding_id, time.time())) - } - } - dump_binding_meta(instance_id, data) + logger.debug(f"*** task bind instance: {instance_id}, before lock") + with ( + new_instance_lock(instance_id), + run_instance_hooks(instance_id, "bind") as (status, output) + ): + logger.debug(f"*** task bind instance: {instance_id}") + backup_instance(instance_id) + data = {"binding_id": binding_id, "credentials": {}, "last_operation": {}} + if status != 0: + data["last_operation"]["state"] = OperationState.FAILED.value + data["last_operation"]["description"] = f"binding {instance_id} failed: {output}" + save_binding_meta(instance_id, data) + return + save_binding_meta(instance_id, data) + chart_path, plan_path = ( + get_chart_path(instance_id), get_plan_path(instance_id)) + shutil.copy(f'{plan_path}/bind.yaml', f'{chart_path}/templates') + values_file = os.path.join(get_plan_path(instance_id), "values.yaml") + args = [ + "template", details.context["instance_name"], chart_path, + "-f", values_file, + "--set", f"fullnameOverride=helmbroker-{details.context['instance_name']}", + "--namespace", details.context["namespace"], + ] + instance_data = load_instance_meta(instance_id) + params = instance_data["details"]["parameters"] + logger.debug(f"helm template parameters: {params}") + args = format_params_to_helm_args(instance_id, params, args) + logger.debug(f"helm template args: {args}") + status, templates = helm(instance_id, *args) # output: templates.yaml + if status != 0: + data["last_operation"]["state"] = OperationState.FAILED.value + data["last_operation"]["description"] = f"binding {instance_id} failed: {templates}" + credential_template = {} + templates = yaml.load_all(templates, Loader=yaml.SafeLoader) + credential_template = next( + (item for item in templates if isinstance(item, dict) and "credential" in item), {} + ) + success_flag = True + errors = [] + for _ in credential_template.get('credential', {}): + if _.get('valueFrom'): + status, val = get_cred_value(details.context["namespace"], _['valueFrom']) + elif _.get('value'): + status, val = 0, _['value'] + else: + status, val = -1, 'invalid value' + if status != 0: + success_flag = False + errors.append(val) + else: + data['credentials'][_['name']] = val + if success_flag: + data['last_operation']['state'] = OperationState.SUCCEEDED.value + data['last_operation']['description'] = ( + f"binding {instance_id} succeeded at {time.time()}") + else: + data['last_operation']['state'] = OperationState.FAILED.value + data['last_operation']['description'] = ( + f"binding {instance_id} failed: {','.join(errors)}") + bind_yaml = f'{chart_path}/templates/bind.yaml' + if os.path.exists(bind_yaml): + os.remove(bind_yaml) + save_binding_meta(instance_id, data) - chart_path = get_chart_path(instance_id) - values_file = os.path.join(get_plan_path(instance_id), "values.yaml") - args = [ - "template", - details.context["instance_name"], - chart_path, - "-f", - values_file, - "--set", - f"fullnameOverride=helmbroker-{details.context['instance_name']}" - ] - status, templates = helm(instance_id, *args) # output: templates.yaml - if status != 0: - data["last_operation"]["state"] = OperationState.FAILED.value - data["last_operation"]["description"] = "binding %s failed: %s" % (instance_id, templates) # noqa - credential_template = yaml.load(templates.split('bind.yaml')[1], Loader=yaml.Loader) # noqa - success_flag = True - errors = [] - for _ in credential_template['credential']: - if _.get('valueFrom'): - status, val = get_cred_value(details.context["namespace"], _['valueFrom']) # noqa - elif _.get('value'): - status, val = 0, _['value'] - else: - status, val = -1, 'invalid value' +@app.task(serializer='pickle') +def unbind(instance_id): + logger.debug(f"*** task unbind instance: {instance_id}, before lock") + with ( + new_instance_lock(instance_id), + run_instance_hooks(instance_id, "unbind") as (status, output) + ): + logger.debug(f"*** task unbind instance: {instance_id}") + backup_instance(instance_id) + data = load_binding_meta(instance_id) if status != 0: - success_flag = False - errors.append(val) - else: - data['credentials'][_['name']] = val - if success_flag: - data['last_operation'] = { - 'state': OperationState.SUCCEEDED.value, - 'description': "binding %s succeeded at %s" % (instance_id, time.time()) # noqa - } - else: - data['last_operation'] = { - 'state': OperationState.FAILED.value, - 'description': "binding %s failed: %s" % (instance_id, ','.join(errors)) # noqa - } - dump_binding_meta(instance_id, data) - bind_yaml = f'{chart_path}/templates/bind.yaml' - if os.path.exists(bind_yaml): - os.remove(bind_yaml) + data['last_operation']['state'] = OperationState.FAILED.value + data['last_operation']['description'] = f"unbind {instance_id} failed: {output}" + save_binding_meta(instance_id, data) + return + binding_file = get_binding_file(instance_id) + if os.path.exists(binding_file): + os.remove(binding_file) + data['last_operation']['state'] = OperationState.SUCCEEDED.value + data['last_operation']['description'] = f"unbind {instance_id} succeeded at {time.time()}" + save_binding_meta(instance_id, data) -@app.task() +@app.task(serializer='pickle') def deprovision(instance_id: str): - with InstanceLock(instance_id): - shutil.copy(get_instance_file(instance_id), "%s.%s" % ( - get_instance_file(instance_id), time.time() - )) + logger.debug(f"*** task deprovision instance: {instance_id}, before lock") + with ( + new_instance_lock(instance_id), + run_instance_hooks(instance_id, "deprovision") as (status, output) + ): + logger.debug(f"*** task deprovision instance: {instance_id}") + backup_instance(instance_id) data = load_instance_meta(instance_id) + if status != 0: + data["last_operation"]["operation"] = "deprovision" + data["last_operation"]["state"] = OperationState.FAILED.value + data["last_operation"]["description"] = f"deprovision {instance_id} failed: {output}" + save_instance_meta(instance_id, data) + return data["last_operation"]["operation"] = "deprovision" data["last_operation"]["state"] = OperationState.IN_PROGRESS.value data["last_operation"]["description"] = ( - "deprovision %s in progress at %s" % (instance_id, time.time())) - dump_instance_meta(instance_id, data) - status, output = helm( - instance_id, - "uninstall", - data["details"]["context"]["instance_name"], - "--namespace", - data["details"]["context"]["namespace"], - ) + f"deprovision {instance_id} in progress at {time.time()}") + save_instance_meta(instance_id, data) + args = [ + "uninstall", data["details"]["context"]["instance_name"], + "--namespace", data["details"]["context"]["namespace"], + ] + logger.debug(f"helm uninstall args: {args}") + status, output = helm(instance_id, *args) if status != 0: data["last_operation"]["state"] = OperationState.FAILED.value - data["last_operation"]["description"] = ( - "deprovision error:\n%s" % output) + data["last_operation"]["description"] = f"deprovision {instance_id} failed: {output}" else: - data["last_operation"]["state"] = ( - OperationState.SUCCEEDED.value) + data["last_operation"]["state"] = OperationState.SUCCEEDED.value data["last_operation"]["description"] = ( - "deprovision succeeded at %s" % time.time()) - dump_instance_meta(instance_id, data) + f"deprovision {instance_id} succeeded at {time.time()}") + save_instance_meta(instance_id, data) diff --git a/rootfs/helmbroker/utils.py b/rootfs/helmbroker/utils.py index dbbf351..e749e85 100644 --- a/rootfs/helmbroker/utils.py +++ b/rootfs/helmbroker/utils.py @@ -1,21 +1,25 @@ import os -import fcntl import yaml import json import subprocess -import time - -from jsonschema import validate -from .config import INSTANCES_PATH, ADDONS_PATH - - +import base64 +import copy +import logging +import jsonschema +from urllib.parse import urlparse, parse_qs +from contextlib import contextmanager +from redis.client import Redis +from redis.sentinel import Sentinel +from .config import VALKEY_URL + +logger = logging.getLogger(__name__) REGISTRY_CONFIG_SUFFIX = '.config/helm/registry.json' REPOSITORY_CACHE_SUFFIX = '.cache/helm/repository' REPOSITORY_CONFIG_SUFFIX = '.config/helm/repository' def command(cmd, *args, output_type="text"): - status, output = subprocess.getstatusoutput("%s %s" % (cmd, " ".join(args))) # noqa + status, output = subprocess.getstatusoutput("%s %s" % (cmd, " ".join(args))) if output_type == "yaml": return yaml.load(output, Loader=yaml.Loader) elif output_type == "json": @@ -23,13 +27,8 @@ def command(cmd, *args, output_type="text"): return status, output -get_instance_path = lambda instance_id: os.path.join(INSTANCES_PATH, instance_id) # noqa -get_instance_file = lambda instance_id: os.path.join(get_instance_path(instance_id), "instance.json") # noqa -get_chart_path = lambda instance_id: os.path.join(get_instance_path(instance_id), "chart") # noqa -get_plan_path = lambda instance_id: os.path.join(get_instance_path(instance_id), "plan") # noqa - - def helm(instance_id, *args, output_type="text"): + from .database.query import get_instance_path instance_path = get_instance_path(instance_id) new_args = [] new_args.extend(args) @@ -41,233 +40,181 @@ def helm(instance_id, *args, output_type="text"): "--repository-config", os.path.join(instance_path, REPOSITORY_CONFIG_SUFFIX), ]) - return command("helm", *args, output_type=output_type) - - -INSTANCE_META_SCHEMA = { - "type": "object", - "properties": { - "id": {"type": "string"}, - "details": { - "type": "object", - "properties": { - "service_id": {"type": "string"}, - "plan_id": {"type": "string"}, - "context": {"type": "object"}, - "parameters": { - 'oneOf': [{'type': 'object'}, {'type': 'null'}] - }, - }, - "required": [ - "service_id", "plan_id", "context" - ] - }, - "last_operation": { - "type": "object", - "properties": { - "state": {"type": "string"}, - "operation": {"type": "string"}, - "description": {"type": "string"} - } - }, - "last_modified_time": {"type": "number"} - }, -} - - -def load_instance_meta(instance_id): - file = get_instance_file(instance_id) - with open(file) as f: - data = json.load(f) - validate(instance=data, schema=INSTANCE_META_SCHEMA) - return data - - -def dump_instance_meta(instance_id, data): - data["last_modified_time"] = time.time() - file = get_instance_file(instance_id) - validate(instance=data, schema=INSTANCE_META_SCHEMA) - with open(file, "w") as f: - f.write(json.dumps(data, sort_keys=True, indent=2)) - - -BINDING_META_SCHEMA = { - "type": "object", - "properties": { - "id": {"type": "string"}, - "credentials": { - "type": "object", - }, - "last_operation": { - "type": "object", - "properties": { - "state": {"type": "string"}, - "description": {"type": "string"} - } - }, - "last_modified_time": {"type": "number"} - } -} - - -def load_binding_meta(instance_id): - file = os.path.join(get_instance_path(instance_id), "binding.json") - with open(file, 'r') as f: - data = json.loads(f.read()) - validate(instance=data, schema=INSTANCE_META_SCHEMA) - return data - - -def dump_binding_meta(instance_id, data): - data["last_modified_time"] = time.time() - file = os.path.join(get_instance_path(instance_id), "binding.json") - validate(instance=data, schema=INSTANCE_META_SCHEMA) - with open(file, "w") as f: - f.write(json.dumps(data, sort_keys=True, indent=2)) - - -ADDONS_META_SCHEMA = { - "type": "object", - "patternProperties": { - ".*": { - "id": {"type": "string"}, - "name": {"type": "string"}, - "version": {"type": "string"}, - "bindable": {"type": "boolean"}, - "instances_retrievable": {"type": "boolean"}, - "bindings_retrievable": {"type": "boolean"}, - "allow_context_updates": {"type": "boolean"}, - "description": {"type": "string"}, - "tags": {"type": "string"}, - "requires": {"type": "array"}, - "metadata": {"type": "object"}, - "plan_updateable": {"type": "boolean"}, - "dashboard_client": {"type": "object"}, - "plans": { - "type": "object", - "id": {"type": "string"}, - "name": {"type": "string"}, - "description": {"type": "string"}, - "metadata": {"type": "object"}, - "free": {"type": "boolean"}, - "bindable": {"type": "boolean"}, - "binding_rotatable": {"type": "boolean"}, - "plan_updateable": {"type": "boolean"}, - "schemas": {"type": "object"}, - "maximum_polling_duration": {"type": "integer"}, - "maintenance_info": {"type": "object"}, - "required": [ - "id", "name", "description" - ] - }, - "required": [ - "id", "name", "description", "bindable", "version", "plans" - ] - } - } -} - - -def load_addons_meta(): - file = os.path.join(ADDONS_PATH, "addons.json") - with open(file, 'r') as f: - data = json.loads(f.read()) - if not data: - return {} - validate(instance=data, schema=INSTANCE_META_SCHEMA) - return data - - -def dump_addons_meta(data): - file = os.path.join(ADDONS_PATH, "addons.json") - validate(instance=data, schema=INSTANCE_META_SCHEMA) - with open(file, "w") as f: - f.write(json.dumps(data, sort_keys=True, indent=2)) - - -def get_addon_meta(service_id): - services = load_addons_meta() - service = [addon for addon in [addons for _, addons in services.items()] - if addon['id'] == service_id][0] - return service - - -def get_addon_path(service_id, plan_id): - service = get_addon_meta(service_id) - plan = [plan for plan in service['plans'] if plan['id'] == plan_id][0] - service_name = f'{service["name"]}-{service["version"]}' - plan_name = plan['name'] - service_path = f'{ADDONS_PATH}/{service_name}/chart/{service["name"]}' - plan_path = f'{ADDONS_PATH}/{service_name}/plans/{plan_name}' - return service_path, plan_path - - -def get_addon_name(service_id): - service = get_addon_meta(service_id) - return service['name'] - - -def get_addon_updateable(service_id): - service = get_addon_meta(service_id) - return service.get('plan_updateable', False) - - -def get_addon_bindable(service_id): - service = get_addon_meta(service_id) - return service.get('bindable', False) - - -def get_cred_value(ns, source): - if source.get('serviceRef'): - return get_service_key_value(ns, source['serviceRef']) - if source.get('configMapRef'): - return get_config_map_key_value(ns, source['configMapRef']) - if source.get('secretKeyRef'): - return get_secret_key_value(ns, source['secretKeyRef']) - return -1, 'invalid valueFrom' - - -def get_service_key_value(ns, service_ref): - args = [ - "get", "svc", service_ref['name'], "-n", ns, '-o', f"jsonpath=\'{service_ref['jsonpath']}\'", # noqa - ] - return command("kubectl", *args) - - -def get_config_map_key_value(ns, config_map_ref): - args = [ - "get", "cm", config_map_ref['name'], "-n", ns, '-o', f"jsonpath=\'{config_map_ref['jsonpath']}\'", # noqa - ] - return command("kubectl", *args) - - -def get_secret_key_value(ns, secret_ref): - args = [ - "get", "secret", secret_ref['name'], "-n", ns, '-o', f"jsonpath=\'{secret_ref['jsonpath']}\'", # noqa - ] - status, output = command("kubectl", *args) - if status == 0: - import base64 - output = base64.b64decode(output).decode() - return status, output - - -class InstanceLock(object): - - def __init__(self, instance_id): - self.instance_id = instance_id - - def __enter__(self): - self.fileno = open( - os.path.join(INSTANCES_PATH, self.instance_id, "instance.lock"), - "w" + return command("helm", *new_args, output_type=output_type) + + +def get_valkey_client(): + url = urlparse(VALKEY_URL) + query = parse_qs(url.query) + if 'master_set' in query: + user, host = url.netloc.split("@") + password = user.split(":")[1] + sentinel = Sentinel( + [host.split(":")], + sentinel_kwargs={'password': password}, + password=password, ) - fcntl.flock(self.fileno, fcntl.LOCK_EX) - return self - - def __exit__(self, exc_type, exc_value, traceback): - fcntl.flock(self.fileno, fcntl.LOCK_UN) - - def __del__(self): - if hasattr(self, "fileno"): - fcntl.flock(self.fileno, fcntl.LOCK_UN) + return sentinel.master_for(query['master_set'][0], socket_timeout=1) + return Redis.from_url(VALKEY_URL) + + +def new_instance_lock(instance_id): + return get_valkey_client().lock(instance_id) + + +@contextmanager +def run_instance_hooks(instance_id, stage): + if stage not in ["provision", "bind", "unbind", "update", "deprovision"]: + raise ValueError(f"Unknown stage {stage}") + from .database.query import get_hooks_path + from .database.savepoint import save_hooks_result + pre_script_file = os.path.join(get_hooks_path(instance_id), f"pre_{stage}.sh") + post_script_file = os.path.join(get_hooks_path(instance_id), f"post_{stage}.sh") + logger.debug(f"instance hook running: {instance_id}, {instance_id}") + result = [] + try: + if os.path.exists(pre_script_file): + status, output = subprocess.getstatusoutput(pre_script_file) + result.append({"script": pre_script_file, "status": status, "output": output}) + else: + status, output = 0, f"skip running {pre_script_file}" + logger.debug(output) + yield status, output + finally: + if os.path.exists(pre_script_file): + status, output = subprocess.getstatusoutput(post_script_file) + result.append({"script": pre_script_file, "status": status, "output": output}) + else: + logger.debug(f"skip running {post_script_file}") + save_hooks_result(instance_id, result) + logger.debug(f"instance hook completed: {instance_id}, {instance_id}") + + +def verify_parameters(allow_parameters, parameters): + """verify parameters allowed or not""" + def merge_parameters(parameters): + raw_para_keys = [] + if "rawValues" in parameters: + raw_values = yaml.safe_load( + base64.b64decode(parameters["rawValues"])) + raw_para_keys = _raw_values_format_keys(raw_values) + parameters.pop("rawValues") + return set(list(parameters.keys()) + raw_para_keys) + + if not parameters or not allow_parameters: + return "", "" + parameters = merge_parameters(copy.deepcopy(parameters)) + return ( + ",".join(_verify_allow_parameters(allow_parameters, parameters)), + ",".join(_verify_required_parameters(allow_parameters, parameters)), + ) + + +def format_params_to_helm_args(instance_id, parameters, args): + """format helm args""" + from .database.savepoint import save_raw_values + params = copy.deepcopy(parameters) + if params and "rawValues" in params \ + and params.get("rawValues", ""): + values = str(base64.b64decode(params["rawValues"]), "utf-8") + raw_values_file = save_raw_values(instance_id, values) + args.extend(["-f", raw_values_file]) + params.pop("rawValues") + if params: + for k, v in params.items(): + args.extend(["--set", f"{k}={v}"]) + return args + + +def _raw_values_format_keys(raw_values, prefix=''): + """ + {'a': {'b': 1, 'c': {'d': 2, 'e': 3}}, 'f': 4} + -> + ['a.b', 'a.c.d', 'a.c.e', 'a.f'] + """ + keys = [] + for key, value in raw_values.items(): + new_prefix = prefix + '.' + key if prefix else key + if isinstance(value, dict): + keys.extend(_raw_values_format_keys(value, new_prefix)) + else: + keys.append(new_prefix) + return keys + + +def _verify_allow_parameters(allow_parameters, parameters): + error_parameters = set() + for parameter in parameters: + error = True + for allow_parameter in allow_parameters: + if parameter.startswith("%s." % allow_parameter["name"]) \ + or parameter == allow_parameter["name"]: + error = False + break + if error: + error_parameters.add(parameter) + return error_parameters + + +def _verify_required_parameters(allow_parameters, parameters): + error_parameters = set() + for allow_parameter in allow_parameters: + if allow_parameter.get("required", False): + error = True + for parameter in parameters: + if parameter.startswith("%s." % allow_parameter["name"]) \ + or parameter == allow_parameter["name"]: + error = False + break + if error: + error_parameters.add(allow_parameter["name"]) + return error_parameters + + +def verify_parameters_by_plan(instance_id, parameters): + """verify parameters allowed or not""" + if not parameters: + return "" + # read schema file + from .database.query import get_plan_schema_path + schema_file = get_plan_schema_path(instance_id) + try: + with open(schema_file, 'r') as f: + schema = json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + return "" + if not schema: + return "" + # get parameters + if "rawValues" in parameters: + params = yaml.safe_load(base64.b64decode(parameters["rawValues"])) + else: + params = _convert_to_nested_dict(parameters) + # validate schema + try: + jsonschema.validate(params, schema) + except jsonschema.ValidationError as e: + return f"could not validate: {e.message}" + return "" + + +def _convert_to_nested_dict(assignments): + """ + {"a.b.c": "1Gi", "a.b.d": "2Gi"} + -> + {'a': {'b': {'c': '1Gi', 'd': '2Gi'}}} + """ + def set_nested_value(d, keys, value): + if len(keys) == 1: + d[keys[0]] = value + else: + if keys[0] not in d: + d[keys[0]] = {} + set_nested_value(d[keys[0]], keys[1:], value) + result = {} + if isinstance(assignments, dict): + # dict format: {"a.b.c": "1Gi"} + for key_path, value in assignments.items(): + keys = key_path.split('.') + set_nested_value(result, keys, value) + return result diff --git a/rootfs/helmbroker/wsgi.py b/rootfs/helmbroker/wsgi.py index ac89884..206c711 100644 --- a/rootfs/helmbroker/wsgi.py +++ b/rootfs/helmbroker/wsgi.py @@ -1,4 +1,5 @@ import os +import logging from flask import Flask, make_response from openbrokerapi import api, log_util from helmbroker.broker import HelmServiceBroker @@ -17,8 +18,7 @@ def readiness(): if "KUBECONFIG" in os.environ: return "OK" elif "KUBERNETES_SERVICE_PORT" in os.environ and \ - ("KUBERNETES_SERVICE_HOST" in os.environ or - "KUBERNETES_CLUSTER_DOMAIN" in os.environ): + "KUBERNETES_SERVICE_HOST" in os.environ: return "OK" return make_response("kubernetes not available", 500) @@ -27,5 +27,5 @@ def readiness(): catalog_api = api.get_blueprint( HelmServiceBroker(), api.BrokerCredentials(USERNAME, PASSWORD), - log_util.basic_config()) + log_util.basic_config(level=logging.DEBUG if Config.DEBUG else logging.INFO)) application.register_blueprint(catalog_api) diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt index 0c459a0..7810104 100644 --- a/rootfs/requirements.txt +++ b/rootfs/requirements.txt @@ -1,6 +1,7 @@ -PyYAML==6.0 -gunicorn==20.1.0 -openbrokerapi==4.1.2 -requests==2.28.1 -celery==5.2.7 -jsonschema==4.14.0 \ No newline at end of file +PyYAML==6.0.2 +gunicorn==23.0.0 +openbrokerapi==4.7.1 +requests==2.32.5 +celery==5.5.3 +redis==6.4.0 +jsonschema==4.25.1 diff --git a/rootfs/setup.cfg b/rootfs/setup.cfg new file mode 100644 index 0000000..e000ebb --- /dev/null +++ b/rootfs/setup.cfg @@ -0,0 +1,4 @@ +[flake8] +max-line-length = 99 +exclude = api/migrations,templates,venv +max-complexity = 12 \ No newline at end of file diff --git a/rootfs/tests/test_utils.py b/rootfs/tests/test_utils.py new file mode 100644 index 0000000..cf83025 --- /dev/null +++ b/rootfs/tests/test_utils.py @@ -0,0 +1,52 @@ +import unittest +from helmbroker import utils + + +class TestUtils(unittest.TestCase): + + default_allow_parameters = [ + { + "name": "deployment.image", + "required": True, + "description": "deployment.image", + }, + { + "name": "deployment.test1.test2", + "required": False, + "description": "deployment.image", + }, + { + "name": "deployment.test1.test3", + "required": False, + "description": "deployment.image", + }, + ] + + def test_verify_parameters(self): + not_allow_keys, required_keys = utils.verify_parameters( + self.default_allow_parameters, + { + "deployment.image.registry": "registry.drycc.cc", + "deployment.image.repository": "drycc-addons/config-reloader", + "deployment.test1.test2": "test2", + "deployment.test1.test3.test4": "test4", + }, + ) + self.assertEqual(required_keys, '') + self.assertEqual(not_allow_keys, '') + not_allow_keys, required_keys = utils.verify_parameters( + self.default_allow_parameters, + { + "deployment.test1.test3": "deployment.test1.test3", + }, + ) + self.assertEqual(required_keys, 'deployment.image') + self.assertEqual(not_allow_keys, '') + not_allow_keys, required_keys = utils.verify_parameters( + self.default_allow_parameters, + { + "deployment.test1.test9": "deployment.test1.test3", + }, + ) + self.assertEqual(required_keys, 'deployment.image') + self.assertEqual(not_allow_keys, 'deployment.test1.test9') diff --git a/versioning.mk b/versioning.mk index 02ebe33..4396eab 100644 --- a/versioning.mk +++ b/versioning.mk @@ -10,13 +10,13 @@ info: @echo "Immutable tag: ${IMAGE}" @echo "Mutable tag: ${MUTABLE_IMAGE}" -.PHONY: docker-push -docker-push: docker-mutable-push docker-immutable-push +.PHONY: podman-push +podman-push: podman-mutable-push podman-immutable-push -.PHONY: docker-immutable-push -docker-immutable-push: - docker push ${IMAGE} +.PHONY: podman-immutable-push +podman-immutable-push: + podman push ${IMAGE} -.PHONY: docker-mutable-push -docker-mutable-push: - docker push ${MUTABLE_IMAGE} +.PHONY: podman-mutable-push +podman-mutable-push: + podman push ${MUTABLE_IMAGE}