From d7e71fb0efa61c35b033bc9e3407ec46ca8c8be9 Mon Sep 17 00:00:00 2001 From: lijianguo Date: Thu, 9 May 2024 18:20:26 +0800 Subject: [PATCH 01/28] chore(helmbroker): bind use namespace --- rootfs/helmbroker/tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rootfs/helmbroker/tasks.py b/rootfs/helmbroker/tasks.py index 380d306..2e6f6e6 100644 --- a/rootfs/helmbroker/tasks.py +++ b/rootfs/helmbroker/tasks.py @@ -164,7 +164,9 @@ def bind(instance_id: str, "-f", values_file, "--set", - f"fullnameOverride=helmbroker-{details.context['instance_name']}" + f"fullnameOverride=helmbroker-{details.context['instance_name']}", + "--namespace", + details.context["namespace"], ] instance_data = load_instance_meta(instance_id) paras = instance_data["details"]["parameters"] From bb95893b3d23bb0a29e427a7837ce3a2e4448bb7 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 31 Jul 2024 15:48:15 +0800 Subject: [PATCH 02/28] feat(database): add redis cache --- charts/helmbroker/Chart.yaml | 5 +- charts/helmbroker/templates/_helpers.tpl | 31 +- .../templates/helmbroker-cronjob-daily.yaml | 1 + .../templates/helmbroker-deployment.yaml | 20 +- charts/helmbroker/values.yaml | 16 +- rootfs/Dockerfile | 2 +- rootfs/helmbroker/broker.py | 70 ++--- rootfs/helmbroker/celery.py | 2 +- rootfs/helmbroker/cleaner.py | 3 +- rootfs/helmbroker/config.py | 17 +- rootfs/helmbroker/database/__init__.py | 0 rootfs/helmbroker/database/fetch.py | 100 ++++++ rootfs/helmbroker/database/metadata.py | 187 +++++++++++ rootfs/helmbroker/database/query.py | 104 +++++++ rootfs/helmbroker/database/savepoint.py | 54 ++++ rootfs/helmbroker/gunicorn/config.py | 4 +- rootfs/helmbroker/loader.py | 120 ------- rootfs/helmbroker/tasks.py | 293 ++++++++---------- rootfs/helmbroker/utils.py | 285 +---------------- rootfs/requirements.txt | 1 + rootfs/setup.cfg | 4 + 21 files changed, 692 insertions(+), 627 deletions(-) create mode 100644 rootfs/helmbroker/database/__init__.py create mode 100644 rootfs/helmbroker/database/fetch.py create mode 100644 rootfs/helmbroker/database/metadata.py create mode 100644 rootfs/helmbroker/database/query.py create mode 100644 rootfs/helmbroker/database/savepoint.py delete mode 100644 rootfs/helmbroker/loader.py create mode 100644 rootfs/setup.cfg diff --git a/charts/helmbroker/Chart.yaml b/charts/helmbroker/Chart.yaml index 4ef1dc8..da3007f 100644 --- a/charts/helmbroker/Chart.yaml +++ b/charts/helmbroker/Chart.yaml @@ -6,8 +6,11 @@ dependencies: - name: common repository: oci://registry.drycc.cc/charts version: ~1.1.2 + - name: redis + repository: oci://registry.drycc.cc/charts + version: x.x.x - name: rabbitmq - repository: oci://registry.drycc.cc/charts-testing + repository: oci://registry.drycc.cc/charts version: x.x.x description: Drycc Workflow helmbroker. maintainers: diff --git a/charts/helmbroker/templates/_helpers.tpl b/charts/helmbroker/templates/_helpers.tpl index b99e10a..39282a7 100644 --- a/charts/helmbroker/templates/_helpers.tpl +++ b/charts/helmbroker/templates/_helpers.tpl @@ -3,26 +3,43 @@ env: - name: "TZ" value: {{ .Values.time_zone | default "UTC" | quote }} -- name: USERNAME +- name: HELMBROKER_USERNAME value: {{ if .Values.username | default "" | ne "" }}{{ .Values.username }}{{ else }}{{ randAlphaNum 32 }}{{ end }} -- name: PASSWORD +- name: HELMBROKER_PASSWORD value: {{ if .Values.password | default "" | ne "" }}{{ .Values.password }}{{ else }}{{ randAlphaNum 32 }}{{ end }} {{- if (.Values.rabbitmqUrl) }} -- name: DRYCC_RABBITMQ_URL +- name: HELMBROKER_RABBITMQ_URL value: {{ .Values.rabbitmqUrl }} {{- else if eq .Values.global.rabbitmqLocation "on-cluster" }} -- name: "DRYCC_RABBITMQ_USERNAME" +- name: "HELMBROKER_RABBITMQ_USERNAME" valueFrom: secretKeyRef: name: rabbitmq-creds key: username -- name: "DRYCC_RABBITMQ_PASSWORD" +- name: "HELMBROKER_RABBITMQ_PASSWORD" valueFrom: secretKeyRef: name: rabbitmq-creds key: password -- name: "DRYCC_RABBITMQ_URL" - value: "amqp://$(DRYCC_RABBITMQ_USERNAME):$(DRYCC_RABBITMQ_PASSWORD)@drycc-rabbitmq.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:5672/drycc" +- name: "HELMBROKER_RABBITMQ_URL" + value: "amqp://$(HELMBROKER_RABBITMQ_USERNAME):$(HELMBROKER_RABBITMQ_PASSWORD)@drycc-rabbitmq.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:5672/drycc" +{{- end }} +{{- if (.Values.redisUrl) }} +- name: HELMBROKER_REDIS_URL + value: {{ .Values.redisUrl }} +{{- else if eq .Values.global.redisLocation "on-cluster" }} +- name: "HELMBROKER_REDIS_ADDRS" + valueFrom: + secretKeyRef: + name: redis-creds + key: addrs +- name: "HELMBROKER_REDIS_PASSWORD" + valueFrom: + secretKeyRef: + name: redis-creds + key: password +- name: "HELMBROKER_REDIS_URL" + value: "redis://:$(HELMBROKER_REDIS_PASSWORD)@$(HELMBROKER_REDIS_ADDRS)/0" {{- end }} {{- range $key, $value := .Values.environment }} - name: {{ $key }} diff --git a/charts/helmbroker/templates/helmbroker-cronjob-daily.yaml b/charts/helmbroker/templates/helmbroker-cronjob-daily.yaml index 909da7d..825e3c8 100644 --- a/charts/helmbroker/templates/helmbroker-cronjob-daily.yaml +++ b/charts/helmbroker/templates/helmbroker-cronjob-daily.yaml @@ -29,6 +29,7 @@ spec: - /bin/bash - -c - python -m helmbroker.cleaner + python -m helmbroker.database.fetch {{- end }} {{- include "helmbroker.envs" . | indent 12 }} {{- include "helmbroker.volumeMounts" . | indent 12 }} diff --git a/charts/helmbroker/templates/helmbroker-deployment.yaml b/charts/helmbroker/templates/helmbroker-deployment.yaml index ee1fa74..7a711ed 100644 --- a/charts/helmbroker/templates/helmbroker-deployment.yaml +++ b/charts/helmbroker/templates/helmbroker-deployment.yaml @@ -27,15 +27,6 @@ spec: nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.api.nodeAffinityPreset.type "key" .Values.api.nodeAffinityPreset.key "values" .Values.api.nodeAffinityPreset.values ) | nindent 10 }} serviceAccount: drycc-helmbroker initContainers: - - name: loader - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/helmbroker:{{.Values.imageTag}} - imagePullPolicy: {{.Values.imagePullPolicy}} - args: - - /bin/bash - - -c - - python -m helmbroker.loader - {{- include "helmbroker.envs" . | indent 10 }} - {{- include "helmbroker.volumeMounts" . | indent 10 }} - name: drycc-helmbroker-init image: registry.drycc.cc/drycc/python-dev:latest imagePullPolicy: {{.Values.imagePullPolicy}} @@ -43,8 +34,17 @@ spec: - netcat - -v - -u - - $(DRYCC_RABBITMQ_URL) + - $(HELMBROKER_REDIS_URL),$(HELMBROKER_RABBITMQ_URL) {{- include "helmbroker.envs" . | indent 10 }} + - name: drycc-helmbroker-fetch + image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/helmbroker:{{.Values.imageTag}} + imagePullPolicy: {{.Values.imagePullPolicy}} + args: + - /bin/bash + - -c + - python -m helmbroker.database.fetch + {{- include "helmbroker.envs" . | indent 10 }} + {{- include "helmbroker.volumeMounts" . | indent 10 }} containers: - name: drycc-helmbroker image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/helmbroker:{{.Values.imageTag}} diff --git a/charts/helmbroker/values.yaml b/charts/helmbroker/values.yaml index 2e457b9..d8c5375 100644 --- a/charts/helmbroker/values.yaml +++ b/charts/helmbroker/values.yaml @@ -33,6 +33,9 @@ celeryReplicas: 1 username: admin password: admin +# Configuring this will no longer use the built-in redis component +redisUrl: "" + # Configuring this will no longer use the built-in rabbitmq component rabbitmqUrl: "" @@ -40,8 +43,8 @@ rabbitmqUrl: "" # can be specified as key-value pairs under environment # this is usually a non required setting. environment: - # DRYCC_DEBUG: true - # HELMBROKER_ROOT: /etc/helmbroker + # HELMBROKER_DEBUG: true + # HELMBROKER_CONFIG_ROOT: /etc/helmbroker api: nodeAffinityPreset: @@ -73,6 +76,10 @@ celery: extraMatchLabels: app: "drycc-helmbroker-celery" +# drycc redis replicas must always be set to 1 +redis: + replicas: 1 + # Default override of addon values addonValues: {} @@ -84,6 +91,11 @@ persistence: volumeName: "" global: + # Set the location of Workflow's redis instance + # Valid values are: + # - on-cluster: Run Redis within the Kubernetes cluster + # - off-cluster: Run Redis outside the Kubernetes cluster (configure in controller section) + redisLocation: "on-cluster" # Set the location of Workflow's rabbitmq instance # Valid values are: # - on-cluster: Run Rabbitmq within the Kubernetes cluster diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 81562a6..806f7c7 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -15,7 +15,7 @@ COPY . ${DRYCC_HOME_DIR} WORKDIR ${DRYCC_HOME_DIR} RUN buildDeps='musl-dev' \ - && install-packages ${buildDeps} openssl ca-certificates \ + && install-packages ${buildDeps} patch openssl ca-certificates \ && install-stack python ${PYTHON_VERSION} \ && install-stack helm ${HELM_VERSION} \ && install-stack kubectl ${KUBECTL_VERSION} && . init-stack \ diff --git a/rootfs/helmbroker/broker.py b/rootfs/helmbroker/broker.py index a8dfe75..561299e 100644 --- a/rootfs/helmbroker/broker.py +++ b/rootfs/helmbroker/broker.py @@ -6,17 +6,19 @@ from openbrokerapi.catalog import ServicePlan from openbrokerapi.errors import ErrInstanceAlreadyExists, ErrAsyncRequired, \ ErrBindingAlreadyExists, ErrBadRequest, ErrInstanceDoesNotExist, \ - ServiceException + ServiceException, ErrBindingDoesNotExist from openbrokerapi.service_broker import ServiceBroker, Service, \ ProvisionDetails, ProvisionedServiceSpec, ProvisionState, GetBindingSpec, \ BindDetails, Binding, BindState, UnbindDetails, UnbindSpec, \ UpdateDetails, UpdateServiceSpec, DeprovisionDetails, \ DeprovisionServiceSpec, LastOperation, OperationState -from .utils import get_instance_path, get_chart_path, get_plan_path, \ - get_addon_path, get_addon_updateable, get_addon_bindable, InstanceLock, \ - load_instance_meta, load_binding_meta, load_addons_meta, \ - get_addon_allow_paras, verify_parameters, get_addon_archive +from .utils import verify_parameters, new_instance_lock +from .database.fetch import fetch_chart_plan +from .database.query import get_instance_path, get_chart_path, get_plan_path, \ + get_addon_updateable, get_addon_bindable, get_addon_allow_params, \ + get_addon_archive, get_binding_file, get_instance_file +from .database.metadata import load_instance_meta, load_binding_meta, load_addons_meta from .tasks import provision, bind, deprovision, update logger = logging.getLogger(__name__) @@ -48,9 +50,9 @@ def provision(self, if get_addon_archive(details.service_id): raise ErrBadRequest( msg="This addon has archived.") - allow_paras = get_addon_allow_paras(details.service_id) + allow_params = get_addon_allow_params(details.service_id) not_allow_keys, required_keys = verify_parameters( - allow_paras, details.parameters) + allow_params, details.parameters) if not_allow_keys: raise ErrBadRequest( msg="parameters %s does not allowed" % not_allow_keys) @@ -58,12 +60,8 @@ def provision(self, raise ErrBadRequest( msg="required parameters %s not exists" % required_keys) os.makedirs(instance_path, exist_ok=True) - chart_path, plan_path = ( - get_chart_path(instance_id), get_plan_path(instance_id)) - addon_chart_path, addon_plan_path = ( - get_addon_path(details.service_id, details.plan_id)) - shutil.copytree(addon_chart_path, chart_path) - shutil.copytree(addon_plan_path, plan_path) + chart_path, plan_path = get_chart_path(instance_id), get_plan_path(instance_id) + fetch_chart_plan(details.service_id, chart_path, details.plan_id, plan_path) provision.delay(instance_id, details) return ProvisionedServiceSpec(state=ProvisionState.IS_ASYNC) @@ -114,10 +112,9 @@ def unbind(self, async_allowed: bool, **kwargs ) -> UnbindSpec: - instance_path = get_instance_path(instance_id) - binding_info = f'{instance_path}/binding.json' - if os.path.exists(binding_info): - os.remove(binding_info) + binding_file = get_binding_file(instance_id) + if os.path.exists(binding_file): + os.remove(binding_file) return UnbindSpec(is_async=False) def update(self, @@ -133,11 +130,11 @@ def update(self, if not is_plan_updateable: raise ErrBadRequest( msg="Instance %s does not updateable" % instance_id) - allow_paras = get_addon_allow_paras(details.service_id) + allow_params = get_addon_allow_params(details.service_id) logger.debug( f"service instance update parameters: {details.parameters}") not_allow_keys, required_keys = verify_parameters( - allow_paras, details.parameters) + allow_params, details.parameters) if not_allow_keys: raise ErrBadRequest( msg="parameters %s does not allowed" % not_allow_keys) @@ -147,13 +144,8 @@ def update(self, if not async_allowed: raise ErrAsyncRequired() if details.plan_id is not None: - plan_path = get_plan_path(instance_id) - # delete the pre plan - shutil.rmtree(plan_path, ignore_errors=True) - _, addon_plan_path = get_addon_path( - details.service_id, details.plan_id) - # add the new plan - shutil.copytree(addon_plan_path, plan_path) + chart_path, plan_path = get_chart_path(instance_id), get_plan_path(instance_id) + fetch_chart_plan(details.service_id, chart_path, details.plan_id, plan_path) update.delay(instance_id, details) return UpdateServiceSpec(is_async=True) @@ -164,7 +156,7 @@ def deprovision(self, **kwargs) -> DeprovisionServiceSpec: if not os.path.exists(get_instance_path(instance_id)): raise ErrInstanceDoesNotExist() - with InstanceLock(instance_id): + with new_instance_lock(instance_id): data = load_instance_meta(instance_id) operation = data["last_operation"]["operation"] if operation == "provision": @@ -181,11 +173,13 @@ def last_operation(self, operation_data: Optional[str], **kwargs ) -> LastOperation: - data = load_instance_meta(instance_id) - return LastOperation( - OperationState(data["last_operation"]["state"]), - data["last_operation"]["description"] - ) + if os.path.exists(get_instance_file(instance_id)): + data = load_instance_meta(instance_id) + return LastOperation( + OperationState(data["last_operation"]["state"]), + data["last_operation"]["description"] + ) + raise ErrInstanceDoesNotExist() def last_binding_operation(self, instance_id: str, @@ -193,8 +187,10 @@ def last_binding_operation(self, operation_data: Optional[str], **kwargs ) -> LastOperation: - data = load_binding_meta(instance_id) - return LastOperation( - OperationState(data["last_operation"]["state"]), - data["last_operation"]["description"] - ) + if os.path.exists(get_binding_file(instance_id)): + data = load_binding_meta(instance_id) + return LastOperation( + OperationState(data["last_operation"]["state"]), + data["last_operation"]["description"] + ) + raise ErrBindingDoesNotExist() diff --git a/rootfs/helmbroker/celery.py b/rootfs/helmbroker/celery.py index 7923690..e9699d2 100644 --- a/rootfs/helmbroker/celery.py +++ b/rootfs/helmbroker/celery.py @@ -18,7 +18,7 @@ class Config(object): task_time_limit = 30 * 60 worker_max_tasks_per_child = 200 result_expires = 24 * 60 * 60 - broker_url = os.environ.get("DRYCC_RABBITMQ_URL", 'amqp://guest:guest@127.0.0.1:5672/') # noqa + broker_url = os.environ.get("HELMBROKER_RABBITMQ_URL", 'amqp://guest:guest@127.0.0.1:5672/') broker_connection_retry_on_startup = True task_default_queue = 'helmbroker.low' task_default_exchange = 'helmbroker.priority' diff --git a/rootfs/helmbroker/cleaner.py b/rootfs/helmbroker/cleaner.py index eeeb5ab..6cd456a 100644 --- a/rootfs/helmbroker/cleaner.py +++ b/rootfs/helmbroker/cleaner.py @@ -6,7 +6,8 @@ from openbrokerapi.service_broker import OperationState from .config import INSTANCES_PATH -from .utils import get_instance_file, load_instance_meta +from .database.query import get_instance_file +from .database.metadata import load_instance_meta logger = logging.getLogger(__name__) diff --git a/rootfs/helmbroker/config.py b/rootfs/helmbroker/config.py index 682d58a..6def522 100644 --- a/rootfs/helmbroker/config.py +++ b/rootfs/helmbroker/config.py @@ -1,14 +1,17 @@ import os -HELMBROKER_ROOT = os.environ.get("HELMBROKER_ROOT", '/etc/helmbroker') -ADDONS_PATH = os.path.join(HELMBROKER_ROOT, 'addons') -CONFIG_PATH = os.path.join(HELMBROKER_ROOT, 'config') -INSTANCES_PATH = os.path.join(HELMBROKER_ROOT, 'instances') +CONFIG_ROOT = os.environ.get("HELMBROKER_CONFIG_ROOT", '/etc/helmbroker') +ADDONS_PATH = os.path.join(CONFIG_ROOT, 'addons') +CONFIG_PATH = os.path.join(CONFIG_ROOT, 'config') +INSTANCES_PATH = os.path.join(CONFIG_ROOT, 'instances') -USERNAME = os.environ.get('USERNAME') -PASSWORD = os.environ.get('PASSWORD') + +USERNAME = os.environ.get('HELMBROKER_USERNAME') +PASSWORD = os.environ.get('HELMBROKER_PASSWORD') + +REDIS_URL = os.environ.get("HELMBROKER_REDIS_URL", 'redis://localhost:6379/0') class Config: - DEBUG = bool(os.environ.get('DRYCC_DEBUG', True)) + DEBUG = bool(os.environ.get('HELMBROKER_DEBUG', True)) diff --git a/rootfs/helmbroker/database/__init__.py b/rootfs/helmbroker/database/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rootfs/helmbroker/database/fetch.py b/rootfs/helmbroker/database/fetch.py new file mode 100644 index 0000000..4fe252f --- /dev/null +++ b/rootfs/helmbroker/database/fetch.py @@ -0,0 +1,100 @@ +import os +import tarfile +import requests +import yaml +import json +import glob +import shutil +import tempfile +import collections + + +def fetch_addons(repository): + if not repository: + return + temp = tempfile.TemporaryDirectory() + try: + index_name = repository['url'].split('/')[-1] + # download index.yaml + remote_index = requests.get(repository['url']).content.decode(encoding="utf-8") + # new index + with open(f"{temp.name}/{index_name}", 'w') as f: + f.write(remote_index) + remote_index = yaml.load(remote_index, Loader=yaml.Loader) + # save index.yaml addons + download_urls = {} + for addon_name, addon_metas in remote_index.get('entries', {}).items(): + download_urls[addon_name] = {} + for addon_meta in addon_metas: + url = "/".join(repository["url"].split("/")[0:-1]) + meta_name = f'{addon_name}-{addon_meta["version"]}' + addon_tgz_url = f'{url}/{meta_name}.tgz' + _fetch_addon(addon_tgz_url, f'{temp.name}/{meta_name}') + return _read_addons_meta(temp.name) + finally: + temp.cleanup() + + +def fetch_chart_plan(addon_id, chart_path, plan_id, plan_path): + from .query import get_addon_meta + addon_meta = get_addon_meta(addon_id) + addon_plan = [plan for plan in addon_meta['plans'] if plan['id'] == plan_id][0] + temp = tempfile.TemporaryDirectory() + try: + _fetch_addon(addon_meta['url'], temp.name) + shutil.rmtree(chart_path, ignore_errors=True) + shutil.rmtree(plan_path, ignore_errors=True) + shutil.copytree(os.path.join(temp.name, "chart", addon_meta["name"]), chart_path) + shutil.copytree(os.path.join(temp.name, "plans", addon_plan['name']), plan_path) + finally: + temp.cleanup() + + +def _fetch_addon(url, dest): + with tempfile.TemporaryFile(suffix=".tgz") as tgz_file: + tgz_file.write(requests.get(url).content) + tgz_file.flush() + tgz_file.seek(0) + os.makedirs(dest, exist_ok=True) + with tarfile.open(fileobj=tgz_file, mode="r:gz") as tarobj: + for tarinfo in tarobj: + tarobj.extract(tarinfo.name, dest) + filename1 = os.path.join(dest, "meta.yaml") + with open(filename1, "r") as f1: + meta = yaml.load(stream=f1, Loader=yaml.Loader) + meta['url'] = url + meta['version'] = str(meta['version']) + meta['tags'] = [tag.strip() for tag in meta.get('tags').split(',') if tag.strip()] + with open(os.path.join(dest, "meta.json"), "w") as f2: + json.dump(meta, f2) + os.remove(filename1) + + +def _read_addons_meta(addons_path): + addons_meta = collections.OrderedDict() + for metafile in glob.glob(os.path.join(addons_path, "*", "meta.json")): + with open(metafile) as f1: + meta = json.load(f1) + meta['plans'] = [] + metapath = os.path.join(os.path.dirname(metafile)) + for planfile in glob.glob(os.path.join(metapath, "plans", "*", "meta.yaml")): + with open(planfile, 'r') as f2: + plan = yaml.load(f2.read(), Loader=yaml.Loader) + meta["plans"].append(plan) + addons_meta[meta['displayName']] = meta + return addons_meta + + +def main(): + from ..config import CONFIG_PATH + from .metadata import save_addons_meta + addons_meta = collections.OrderedDict() + with open(f'{CONFIG_PATH}/repositories', 'r') as f: + repositories = yaml.load(f.read(), Loader=yaml.Loader) + for repository in repositories: + addons_meta.update(fetch_addons(repository)) + save_addons_meta(addons_meta) + + +if __name__ == '__main__': + main() diff --git a/rootfs/helmbroker/database/metadata.py b/rootfs/helmbroker/database/metadata.py new file mode 100644 index 0000000..6b37041 --- /dev/null +++ b/rootfs/helmbroker/database/metadata.py @@ -0,0 +1,187 @@ +import os +import json +import time +import logging +import jsonschema + +from redis import client +from ..config import ADDONS_PATH, REDIS_URL + +logger = logging.getLogger(__name__) + +INSTANCE_META_SCHEMA = { + "type": "object", + "properties": { + "id": {"type": "string"}, + "details": { + "type": "object", + "properties": { + "service_id": {"type": "string"}, + "plan_id": {"type": "string"}, + "context": {"type": "object"}, + "parameters": { + 'oneOf': [{'type': 'object'}, {'type': 'null'}] + }, + }, + "required": [ + "service_id", "plan_id", "context" + ] + }, + "last_operation": { + "type": "object", + "properties": { + "state": {"type": "string"}, + "operation": {"type": "string"}, + "description": {"type": "string"} + } + }, + "last_modified_time": {"type": "number"} + }, +} + +BINDING_META_SCHEMA = { + "type": "object", + "properties": { + "id": {"type": "string"}, + "credentials": { + "type": "object", + }, + "last_operation": { + "type": "object", + "properties": { + "state": {"type": "string"}, + "description": {"type": "string"} + } + }, + "last_modified_time": {"type": "number"} + } +} + +ADDONS_META_SCHEMA = { + "type": "object", + "patternProperties": { + ".*": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "version": {"type": "string"}, + "bindable": {"type": "boolean"}, + "instances_retrievable": {"type": "boolean"}, + "bindings_retrievable": {"type": "boolean"}, + "allow_context_updates": {"type": "boolean"}, + "description": {"type": "string"}, + "tags": {"type": "array", "items": {"type": "string"}}, + "requires": {"type": "array"}, + "metadata": {"type": "object"}, + "plan_updateable": {"type": "boolean"}, + "dashboard_client": {"type": "object"}, + "plans": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "description": {"type": "string"}, + "metadata": {"type": "object"}, + "free": {"type": "boolean"}, + "bindable": {"type": "boolean"}, + "binding_rotatable": {"type": "boolean"}, + "plan_updateable": {"type": "boolean"}, + "schemas": {"type": "object"}, + "maximum_polling_duration": {"type": "integer"}, + "maintenance_info": {"type": "object"}, + }, + "required": ["id", "name", "description"] + }, + "minItems": 1, + + }, + }, + "required": ["id", "name", "description", "bindable", "version", "plans"] + }, + }, + "additionalProperties": False, +} + + +def save_instance_meta(instance_id, data): + cache_key = f"helmbroker:instance:{instance_id}" + from .query import get_instance_file + data["last_modified_time"] = time.time() + file = get_instance_file(instance_id) + jsonschema.validate(instance=data, schema=INSTANCE_META_SCHEMA) + + json_data = json.dumps(data, sort_keys=True, indent=2) + with open(file, "w") as f: + f.write(json_data) + redis = client.Redis.from_url(REDIS_URL) + redis.set(cache_key, json_data) + + +def load_instance_meta(instance_id): + cache_key = f"helmbroker:instance:{instance_id}" + redis = client.Redis.from_url(REDIS_URL) + + json_data = redis.get(cache_key) + if not json_data: + from .query import get_instance_file + file = get_instance_file(instance_id) + with open(file) as f: + json_data = f.read() + redis.set(cache_key, json_data) + return json.loads(json_data) + + +def save_binding_meta(instance_id, data): + from .query import get_binding_file + cache_key = f"helmbroker:binding:{instance_id}" + data["last_modified_time"] = time.time() + file = get_binding_file(instance_id) + jsonschema.validate(instance=data, schema=BINDING_META_SCHEMA) + + json_data = json.dumps(data, sort_keys=True, indent=2) + with open(file, "w") as f: + f.write(json_data) + redis = client.Redis.from_url(REDIS_URL) + redis.set(cache_key, json_data) + + +def load_binding_meta(instance_id): + from .query import get_binding_file + cache_key = f"helmbroker:binding:{instance_id}" + redis = client.Redis.from_url(REDIS_URL) + json_data = redis.get(cache_key) + if not json_data: + file = get_binding_file(instance_id) + with open(file, 'r') as f: + json_data = f.read() + redis.set(cache_key, json_data) + return json.loads(json_data) + + +def save_addons_meta(data): + cache_key = "helmbroker:addons" + os.makedirs(ADDONS_PATH, exist_ok=True) + file = os.path.join(ADDONS_PATH, "addons.json") + jsonschema.validate(instance=data, schema=ADDONS_META_SCHEMA) + + json_data = json.dumps(data, sort_keys=True, indent=2) + with open(file, "w") as f: + f.write(json_data) + redis = client.Redis.from_url(REDIS_URL) + redis.set(cache_key, json_data) + + +def load_addons_meta(): + cache_key = "helmbroker:addons" + redis = client.Redis.from_url(REDIS_URL) + + json_data = redis.get(cache_key) + if not json_data: + file = os.path.join(ADDONS_PATH, "addons.json") + with open(file, 'r') as f: + json_data = f.read() + redis.set(cache_key, json_data) + return json.loads(json_data) diff --git a/rootfs/helmbroker/database/query.py b/rootfs/helmbroker/database/query.py new file mode 100644 index 0000000..b87810b --- /dev/null +++ b/rootfs/helmbroker/database/query.py @@ -0,0 +1,104 @@ +import os +import base64 + +from ..utils import command +from ..config import INSTANCES_PATH +from .metadata import load_addons_meta + + +def get_instance_path(instance_id): + return os.path.join(INSTANCES_PATH, instance_id) + + +def get_instance_file(instance_id): + return os.path.join(get_instance_path(instance_id), "instance.json") + + +def get_chart_path(instance_id): + return os.path.join(get_instance_path(instance_id), "chart") + + +def get_plan_path(instance_id): + return os.path.join(get_instance_path(instance_id), "plan") + + +def get_binding_file(instance_id): + return os.path.join(get_instance_path(instance_id), "binding.json") + + +def get_backups_path(instance_id): + return os.path.join(get_instance_path(instance_id), "backups") + + +def get_addon_values_file(instance_id): + return os.path.join(get_instance_path(instance_id), "addon-values.yaml") + + +def get_custom_addon_values_file(instance_id): + return os.path.join(get_instance_path(instance_id), "custom-addon-values.yaml") + + +def get_addon_updateable(addon_id): + addon_meta = get_addon_meta(addon_id) + return addon_meta.get('plan_updateable', False) + + +def get_addon_bindable(addon_id): + addon_meta = get_addon_meta(addon_id) + return addon_meta.get('bindable', False) + + +def get_addon_allow_params(addon_id): + addon_meta = get_addon_meta(addon_id) + return addon_meta.get('allow_parameters', []) + + +def get_addon_archive(addon_id): + addon_meta = get_addon_meta(addon_id) + return addon_meta.get('archive', False) + + +def get_cred_value(ns, source): + if source.get('serviceRef'): + return _get_service_key_value(ns, source['serviceRef']) + if source.get('configMapRef'): + return _get_config_map_key_value(ns, source['configMapRef']) + if source.get('secretKeyRef'): + return _get_secret_key_value(ns, source['secretKeyRef']) + return -1, 'invalid valueFrom' + + +def get_addon_meta(addon_id): + addons_meta = load_addons_meta() + addons_meta = [ + addon for addon in [addons for _, addons in addons_meta.items()] + if addon['id'] == addon_id + ] + return addons_meta[0] if len(addons_meta) > 0 else None + + +def _get_service_key_value(ns, service_ref): + args = [ + "get", "svc", service_ref['name'], "-n", ns, + '-o', f"jsonpath=\'{service_ref['jsonpath']}\'", + ] + return command("kubectl", *args) + + +def _get_config_map_key_value(ns, config_map_ref): + args = [ + "get", "cm", config_map_ref['name'], "-n", ns, + '-o', f"jsonpath=\'{config_map_ref['jsonpath']}\'", + ] + return command("kubectl", *args) + + +def _get_secret_key_value(ns, secret_ref): + args = [ + "get", "secret", secret_ref['name'], "-n", ns, '-o', + f"jsonpath=\'{secret_ref['jsonpath']}\'", + ] + status, output = command("kubectl", *args) + if status == 0: + output = base64.b64decode(output).decode() + return status, output diff --git a/rootfs/helmbroker/database/savepoint.py b/rootfs/helmbroker/database/savepoint.py new file mode 100644 index 0000000..2467349 --- /dev/null +++ b/rootfs/helmbroker/database/savepoint.py @@ -0,0 +1,54 @@ +import os +import logging +import yaml +import shutil +import datetime + +from ..config import CONFIG_PATH +from .query import get_instance_path, get_backups_path, get_addon_meta, get_addon_values_file, \ + get_custom_addon_values_file + +logger = logging.getLogger(__name__) + + +def save_raw_values(instance_id, data): + file = get_custom_addon_values_file(instance_id) + with open(file, "w") as f: + f.write(data) + return file + + +def save_addon_values(service_id, instance_id): + file = get_addon_values_file(instance_id) + service = get_addon_meta(service_id) + logger.debug(f"save_addon_values service: {service}") + if not os.path.exists(f'{CONFIG_PATH}/addon-values'): + return None + with open(file, "w") as fw: + with open(f'{CONFIG_PATH}/addon-values', 'r') as f: + addons_values = yaml.load(f.read(), Loader=yaml.Loader) + logger.debug(f"save_addon_values addons_values: {addons_values}") + addon_values = addons_values.get(service["name"], {}).\ + get(service["version"], {}) + logger.debug(f"save_addon_values addon_values: {addon_values}") + if not addon_values: + return None + fw.write(yaml.dump(addon_values)) + return file + + +def backup_instance(instance_id): + now = datetime.datetime.now(datetime.timezone.utc) + backup_path = os.path.join(get_backups_path(instance_id), now.isoformat()) + os.makedirs(backup_path, exist_ok=True) + + addon_values_file = get_addon_values_file(instance_id) + if os.path.exists(addon_values_file): + shutil.copy(addon_values_file, backup_path) + custom_addon_values_file = get_custom_addon_values_file(instance_id) + if os.path.exists(custom_addon_values_file): + shutil.copy(custom_addon_values_file, backup_path) + + instance_path = get_instance_path(instance_id) + shutil.copytree(os.path.join(instance_path, "plan"), os.path.join(backup_path, "plan")) + shutil.copytree(os.path.join(instance_path, "chart"), os.path.join(backup_path, "chart")) diff --git a/rootfs/helmbroker/gunicorn/config.py b/rootfs/helmbroker/gunicorn/config.py index 4038c74..716f9e5 100644 --- a/rootfs/helmbroker/gunicorn/config.py +++ b/rootfs/helmbroker/gunicorn/config.py @@ -18,14 +18,14 @@ def worker_int(worker): - """Print a stack trace when a worker receives a SIGINT or SIGQUIT signal.""" # noqa + """Print a stack trace when a worker receives a SIGINT or SIGQUIT signal.""" worker.log.warning('worker terminated') import traceback traceback.print_stack() def worker_abort(worker): - """Print a stack trace when a worker receives a SIGABRT signal, generally on timeout.""" # noqa + """Print a stack trace when a worker receives a SIGABRT signal, generally on timeout.""" worker.log.warning('worker aborted') import traceback traceback.print_stack() diff --git a/rootfs/helmbroker/loader.py b/rootfs/helmbroker/loader.py deleted file mode 100644 index abdec39..0000000 --- a/rootfs/helmbroker/loader.py +++ /dev/null @@ -1,120 +0,0 @@ -import os -import shutil -import tarfile -import requests -import yaml - -from .config import ADDONS_PATH, CONFIG_PATH -from .utils import dump_addons_meta - - -def download_file(url, dest): - if not os.path.exists(dest): - os.system(f'mkdir -p {dest}') - filename = url.split('/')[-1] - file = requests.get(url) - with open(f"{dest}/{filename}", 'wb') as f: - f.write(file.content) - if filename.endswith(".yaml") or filename.endswith(".yml"): - return yaml.load(file.content.decode(encoding="utf-8"), - Loader=yaml.Loader) - - -def read_file(filename): - if not os.path.exists(filename): - return - with open(filename, 'r') as f: - file_content = f.read() - return file_content - - -def save_file(content, dest, filename): - if not os.path.exists(dest): - os.system(f'mkdir -p {dest}') - with open(f"{dest}/{filename}", 'w') as f: - f.write(content) - - -def extract_tgz(tgz_file, dest): - if not os.path.exists(tgz_file): - return - if not os.path.exists(dest): - os.system(f'mkdir -p {dest}') - - tarobj = tarfile.open(tgz_file, "r:gz") - for tarinfo in tarobj: - tarobj.extract(tarinfo.name, dest) - tarobj.close() - - -def addons_meta_file(): - meta_files = [] - # get meta.yaml - for root, dirnames, filenames in os.walk(ADDONS_PATH): - for filename in filenames: - if filename == 'meta.yaml': - meta_files.append(os.path.join(root, filename)) - meta_files = [meta_file.split(ADDONS_PATH)[1] for meta_file in meta_files] - addons_meta = [] - plans_meta = [] - for meta_file in meta_files: - if len(meta_file.split('/')) == 3: - addons_meta.append(meta_file.split('/')[1:]) - else: - plans_meta.append(meta_file.split('/')[1:]) - addons_dict = {} - for addon_meta in addons_meta: - with open(f'{ADDONS_PATH}/{"/".join(addon_meta)}', 'r') as f: - meta = yaml.load(f.read(), Loader=yaml.Loader) - meta['tags'] = meta.get('tags').split(', ') if meta.get('tags') else [] # noqa - meta['plans'] = [] - addons_dict[meta['displayName']] = meta - addon_plans_meta = [] - for plan_meta in plans_meta: - if plan_meta[0] == meta['displayName']: - addon_plans_meta.append(plan_meta) - elif f'{"-".join(plan_meta[0].split("-")[0:-1])}' == meta['displayName']: # noqa - addon_plans_meta.append(plan_meta) - for addon_plan_meta in addon_plans_meta: - with open(f'{ADDONS_PATH}/{"/".join(addon_plan_meta)}', 'r') as f: - addons_mata = yaml.load(f.read(), Loader=yaml.Loader) - addons_dict[meta['displayName']]['plans'].append(addons_mata) # noqa - dump_addons_meta(addons_dict) - - -def load_addons(repository): - if not repository: - return - index_name = repository['url'].split('/')[-1] - local_index_file = f'{ADDONS_PATH}/{index_name}' - # download index.yaml - remote_index = requests.get(repository['url']).content.decode( - encoding="utf-8") - # compare index.yaml, is update - local_index = read_file(local_index_file) - if local_index and remote_index == local_index: - return - # delete old repository catalog - if os.path.exists(ADDONS_PATH): - shutil.rmtree(ADDONS_PATH, ignore_errors=True) - else: - os.makedirs(ADDONS_PATH, exist_ok=True) - # new index - save_file(remote_index, ADDONS_PATH, index_name) - remote_index = yaml.load(remote_index, Loader=yaml.Loader) - # save index.yaml addons - for addon_name, v in remote_index.get('entries', {}).items(): - for _ in v: - url = "/".join(repository["url"].split("/")[0:-1]) - tgz_name = f'{addon_name}-{_["version"]}' - addon_tgz_url = f'{url}/{tgz_name}.tgz' - download_file(addon_tgz_url, ADDONS_PATH) - extract_tgz(f'{ADDONS_PATH}/{tgz_name}.tgz', - f'{ADDONS_PATH}/{tgz_name}') - addons_meta_file() - - -if __name__ == '__main__': - with open(f'{CONFIG_PATH}/repositories', 'r') as f: - repositories = yaml.load(f.read(), Loader=yaml.Loader) - load_addons(repositories[0]) diff --git a/rootfs/helmbroker/tasks.py b/rootfs/helmbroker/tasks.py index 2e6f6e6..22c3e94 100644 --- a/rootfs/helmbroker/tasks.py +++ b/rootfs/helmbroker/tasks.py @@ -1,6 +1,5 @@ import os import time -import shutil import yaml import logging @@ -8,31 +7,30 @@ UpdateDetails, BindDetails from .celery import app -from .utils import get_plan_path, get_chart_path, get_cred_value, \ - InstanceLock, dump_instance_meta, dump_binding_meta, load_instance_meta, \ - get_instance_file, helm, dump_addon_values, format_paras_to_helm_args +from .utils import helm, format_params_to_helm_args, new_instance_lock + +from .database.metadata import save_instance_meta, save_binding_meta, load_instance_meta +from .database.savepoint import save_addon_values, backup_instance +from .database.query import get_plan_path, get_chart_path, get_cred_value logger = logging.getLogger(__name__) @app.task(serializer='pickle') def provision(instance_id: str, details: ProvisionDetails): - with InstanceLock(instance_id): + with new_instance_lock(instance_id): + backup_instance(instance_id) # create instance.json - dump_instance_meta(instance_id, { + save_instance_meta(instance_id, { "id": instance_id, "details": { - "service_id": details.service_id, - "plan_id": details.plan_id, + "service_id": details.service_id, "plan_id": details.plan_id, "context": details.context, "parameters": details.parameters if details.parameters else {}, }, "last_operation": { - "state": OperationState.IN_PROGRESS.value, - "operation": "provision", - "description": ( - "provision %s in progress at %s" % ( - instance_id, time.time())) + "state": OperationState.IN_PROGRESS.value, "operation": "provision", + "description": ("provision %s in progress at %s" % (instance_id, time.time())) } }) @@ -41,101 +39,78 @@ def provision(instance_id: str, details: ProvisionDetails): if os.path.exists(bind_yaml): os.remove(bind_yaml) if os.path.exists(f'{chart_path}/Chart.yaml'): - args = [ - "dependency", - "update", - chart_path, - ] + args = ["dependency", "update", chart_path] helm(instance_id, *args) values_file = os.path.join(get_plan_path(instance_id), "values.yaml") args = [ - "install", - details.context["instance_name"], - chart_path, - "--namespace", - details.context["namespace"], - "--create-namespace", - "--wait", - "--timeout", - "25m0s", - "-f", - values_file, - "--set", - f"fullnameOverride=helmbroker-{details.context['instance_name']}" + "install", details.context["instance_name"], chart_path, + "--namespace", details.context["namespace"], "--create-namespace", + "--wait", "--timeout", "25m0s", "-f", values_file, + "--set", f"fullnameOverride=helmbroker-{details.context['instance_name']}" ] - addon_values_file = dump_addon_values(details.service_id, instance_id) + addon_values_file = save_addon_values(details.service_id, instance_id) if addon_values_file: args.insert(9, "-f") args.insert(10, addon_values_file) logger.debug(f"helm install parameters :{details.parameters}") - args = format_paras_to_helm_args(instance_id, details.parameters, args) + args = format_params_to_helm_args(instance_id, details.parameters, args) logger.debug(f"helm install args:{args}") status, output = helm(instance_id, *args) data = load_instance_meta(instance_id) if status != 0: data["last_operation"]["state"] = OperationState.FAILED.value - data["last_operation"]["description"] = ( - "provision error:\n%s" % output) + data["last_operation"]["description"] = "provision error:\n%s" % output else: data["last_operation"]["state"] = OperationState.SUCCEEDED.value - data["last_operation"]["description"] = ( - "provision succeeded at %s" % time.time()) - dump_instance_meta(instance_id, data) + data["last_operation"]["description"] = "provision succeeded at %s" % time.time() + save_instance_meta(instance_id, data) @app.task(serializer='pickle') def update(instance_id: str, details: UpdateDetails): - data = load_instance_meta(instance_id) - if details.service_id: - data['details']['service_id'] = details.service_id - if details.plan_id: - data['details']['service_id'] = details.plan_id - if details.context: - data['details']['context'] = details.context - if details.parameters: - paras = data['details']['parameters'] - paras.update(details.parameters) - # remove the key which value is null - data['details']['parameters'] = {k: v for k, v in paras.items() if v != ""} # noqa - data['last_operation']["state"] = OperationState.IN_PROGRESS.value - data['last_operation']["description"] = "update %s in progress at %s" % (instance_id, time.time()) # noqa - dump_instance_meta(instance_id, data) - chart_path = get_chart_path(instance_id) - values_file = os.path.join(get_plan_path(instance_id), "values.yaml") - args = [ - "upgrade", - details.context["instance_name"], - chart_path, - "--namespace", - details.context["namespace"], - "--create-namespace", - "--wait", - "--timeout", - "25m0s", - "--reuse-values", - "-f", - values_file, - "--set", - f"fullnameOverride=helmbroker-{details.context['instance_name']}" - ] - addon_values_file = dump_addon_values(details.service_id, instance_id) - if addon_values_file: - args.insert(10, "-f") - args.insert(11, addon_values_file) - paras = data['details']['parameters'] - logger.debug(f"helm upgrade parameters: {paras}") - args = format_paras_to_helm_args(instance_id, paras, args) - logger.debug(f"helm upgrade args:{args}") - status, output = helm(instance_id, *args) - if status != 0: - data["last_operation"]["state"] = OperationState.FAILED.value - data["last_operation"]["description"] = ( - "update %s failed: %s" % (instance_id, output)) - else: - data["last_operation"]["state"] = OperationState.SUCCEEDED.value - data["last_operation"]["description"] = ( - "update %s succeeded at %s" % (instance_id, time.time())) - dump_instance_meta(instance_id, data) + with new_instance_lock(instance_id): + backup_instance(instance_id) + data = load_instance_meta(instance_id) + if details.service_id: + data['details']['service_id'] = details.service_id + if details.plan_id: + data['details']['service_id'] = details.plan_id + if details.context: + data['details']['context'] = details.context + if details.parameters: + params = data['details']['parameters'] + params.update(details.parameters) + # remove the key which value is null + data['details']['parameters'] = {k: v for k, v in params.items() if v != ""} + data['last_operation']["state"] = OperationState.IN_PROGRESS.value + data['last_operation']["description"] = "update %s in progress at %s" % ( + instance_id, time.time()) + save_instance_meta(instance_id, data) + chart_path = get_chart_path(instance_id) + values_file = os.path.join(get_plan_path(instance_id), "values.yaml") + args = [ + "upgrade", details.context["instance_name"], chart_path, + "--namespace", details.context["namespace"], "--create-namespace", + "--wait", "--timeout", "25m0s", "--reuse-values", "-f", values_file, + "--set", f"fullnameOverride=helmbroker-{details.context['instance_name']}" + ] + addon_values_file = save_addon_values(details.service_id, instance_id) + if addon_values_file: + args.insert(10, "-f") + args.insert(11, addon_values_file) + params = data['details']['parameters'] + logger.debug(f"helm upgrade parameters: {params}") + args = format_params_to_helm_args(instance_id, params, args) + logger.debug(f"helm upgrade args:{args}") + status, output = helm(instance_id, *args) + if status != 0: + data["last_operation"]["state"] = OperationState.FAILED.value + data["last_operation"]["description"] = "update %s failed: %s" % (instance_id, output) + else: + data["last_operation"]["state"] = OperationState.SUCCEEDED.value + data["last_operation"]["description"] = ( + "update %s succeeded at %s" % (instance_id, time.time())) + save_instance_meta(instance_id, data) @app.task(serializer='pickle') @@ -144,98 +119,86 @@ def bind(instance_id: str, details: BindDetails, async_allowed: bool, **kwargs): - data = { - "binding_id": binding_id, - "credentials": {}, - "last_operation": { - "state": OperationState.IN_PROGRESS.value, - "description": ( - "binding %s in progress at %s" % (binding_id, time.time())) + with new_instance_lock(instance_id): + backup_instance(instance_id) + data = { + "binding_id": binding_id, "credentials": {}, + "last_operation": { + "state": OperationState.IN_PROGRESS.value, + "description": "binding %s in progress at %s" % (binding_id, time.time()) + } } - } - dump_binding_meta(instance_id, data) - - chart_path = get_chart_path(instance_id) - values_file = os.path.join(get_plan_path(instance_id), "values.yaml") - args = [ - "template", - details.context["instance_name"], - chart_path, - "-f", - values_file, - "--set", - f"fullnameOverride=helmbroker-{details.context['instance_name']}", - "--namespace", - details.context["namespace"], - ] - instance_data = load_instance_meta(instance_id) - paras = instance_data["details"]["parameters"] - logger.debug(f"helm template parameters: {paras}") - args = format_paras_to_helm_args(instance_id, paras, args) - logger.debug(f"helm template args: {args}") - status, templates = helm(instance_id, *args) # output: templates.yaml - if status != 0: - data["last_operation"]["state"] = OperationState.FAILED.value - data["last_operation"]["description"] = "binding %s failed: %s" % (instance_id, templates) # noqa - - credential_template = yaml.load(templates.split('bind.yaml')[1], Loader=yaml.Loader) # noqa - success_flag = True - errors = [] - for _ in credential_template['credential']: - if _.get('valueFrom'): - status, val = get_cred_value(details.context["namespace"], _['valueFrom']) # noqa - elif _.get('value'): - status, val = 0, _['value'] - else: - status, val = -1, 'invalid value' + save_binding_meta(instance_id, data) + chart_path = get_chart_path(instance_id) + values_file = os.path.join(get_plan_path(instance_id), "values.yaml") + args = [ + "template", details.context["instance_name"], chart_path, + "-f", values_file, + "--set", f"fullnameOverride=helmbroker-{details.context['instance_name']}", + "--namespace", details.context["namespace"], + ] + instance_data = load_instance_meta(instance_id) + params = instance_data["details"]["parameters"] + logger.debug(f"helm template parameters: {params}") + args = format_params_to_helm_args(instance_id, params, args) + logger.debug(f"helm template args: {args}") + status, templates = helm(instance_id, *args) # output: templates.yaml if status != 0: - success_flag = False - errors.append(val) + data["last_operation"]["state"] = OperationState.FAILED.value + data["last_operation"]["description"] = "binding %s failed: %s" % ( + instance_id, templates) + + credential_template = yaml.load(templates.split('bind.yaml')[1], Loader=yaml.Loader) + success_flag = True + errors = [] + for _ in credential_template['credential']: + if _.get('valueFrom'): + status, val = get_cred_value(details.context["namespace"], _['valueFrom']) + elif _.get('value'): + status, val = 0, _['value'] + else: + status, val = -1, 'invalid value' + if status != 0: + success_flag = False + errors.append(val) + else: + data['credentials'][_['name']] = val + if success_flag: + data['last_operation'] = { + 'state': OperationState.SUCCEEDED.value, + 'description': "binding %s succeeded at %s" % (instance_id, time.time()) + } else: - data['credentials'][_['name']] = val - if success_flag: - data['last_operation'] = { - 'state': OperationState.SUCCEEDED.value, - 'description': "binding %s succeeded at %s" % (instance_id, time.time()) # noqa - } - else: - data['last_operation'] = { - 'state': OperationState.FAILED.value, - 'description': "binding %s failed: %s" % (instance_id, ','.join(errors)) # noqa - } - dump_binding_meta(instance_id, data) - bind_yaml = f'{chart_path}/templates/bind.yaml' - if os.path.exists(bind_yaml): - os.remove(bind_yaml) + data['last_operation'] = { + 'state': OperationState.FAILED.value, + 'description': "binding %s failed: %s" % (instance_id, ','.join(errors)) + } + bind_yaml = f'{chart_path}/templates/bind.yaml' + if os.path.exists(bind_yaml): + os.remove(bind_yaml) + save_binding_meta(instance_id, data) @app.task() def deprovision(instance_id: str): - with InstanceLock(instance_id): - shutil.copy(get_instance_file(instance_id), "%s.%s" % ( - get_instance_file(instance_id), time.time() - )) + with new_instance_lock(instance_id): + backup_instance(instance_id) data = load_instance_meta(instance_id) data["last_operation"]["operation"] = "deprovision" data["last_operation"]["state"] = OperationState.IN_PROGRESS.value data["last_operation"]["description"] = ( "deprovision %s in progress at %s" % (instance_id, time.time())) - dump_instance_meta(instance_id, data) + save_instance_meta(instance_id, data) args = [ - "uninstall", - data["details"]["context"]["instance_name"], - "--namespace", - data["details"]["context"]["namespace"], + "uninstall", data["details"]["context"]["instance_name"], + "--namespace", data["details"]["context"]["namespace"], ] logger.debug(f"helm uninstall args: {args}") status, output = helm(instance_id, *args) if status != 0: data["last_operation"]["state"] = OperationState.FAILED.value - data["last_operation"]["description"] = ( - "deprovision error:\n%s" % output) + data["last_operation"]["description"] = "deprovision error:\n%s" % output else: - data["last_operation"]["state"] = ( - OperationState.SUCCEEDED.value) - data["last_operation"]["description"] = ( - "deprovision succeeded at %s" % time.time()) - dump_instance_meta(instance_id, data) + data["last_operation"]["state"] = OperationState.SUCCEEDED.value + data["last_operation"]["description"] = "deprovision succeeded at %s" % time.time() + save_instance_meta(instance_id, data) diff --git a/rootfs/helmbroker/utils.py b/rootfs/helmbroker/utils.py index a2f6d31..b4271bf 100644 --- a/rootfs/helmbroker/utils.py +++ b/rootfs/helmbroker/utils.py @@ -1,111 +1,23 @@ import os -import fcntl import yaml import json import subprocess -import time import base64 import copy import logging -from jsonschema import validate -from .config import INSTANCES_PATH, ADDONS_PATH, CONFIG_PATH +from redis import client +from .config import REDIS_URL logger = logging.getLogger(__name__) REGISTRY_CONFIG_SUFFIX = '.config/helm/registry.json' REPOSITORY_CACHE_SUFFIX = '.cache/helm/repository' REPOSITORY_CONFIG_SUFFIX = '.config/helm/repository' -INSTANCE_META_SCHEMA = { - "type": "object", - "properties": { - "id": {"type": "string"}, - "details": { - "type": "object", - "properties": { - "service_id": {"type": "string"}, - "plan_id": {"type": "string"}, - "context": {"type": "object"}, - "parameters": { - 'oneOf': [{'type': 'object'}, {'type': 'null'}] - }, - }, - "required": [ - "service_id", "plan_id", "context" - ] - }, - "last_operation": { - "type": "object", - "properties": { - "state": {"type": "string"}, - "operation": {"type": "string"}, - "description": {"type": "string"} - } - }, - "last_modified_time": {"type": "number"} - }, -} -BINDING_META_SCHEMA = { - "type": "object", - "properties": { - "id": {"type": "string"}, - "credentials": { - "type": "object", - }, - "last_operation": { - "type": "object", - "properties": { - "state": {"type": "string"}, - "description": {"type": "string"} - } - }, - "last_modified_time": {"type": "number"} - } -} -ADDONS_META_SCHEMA = { - "type": "object", - "patternProperties": { - ".*": { - "id": {"type": "string"}, - "name": {"type": "string"}, - "version": {"type": "string"}, - "bindable": {"type": "boolean"}, - "instances_retrievable": {"type": "boolean"}, - "bindings_retrievable": {"type": "boolean"}, - "allow_context_updates": {"type": "boolean"}, - "description": {"type": "string"}, - "tags": {"type": "string"}, - "requires": {"type": "array"}, - "metadata": {"type": "object"}, - "plan_updateable": {"type": "boolean"}, - "dashboard_client": {"type": "object"}, - "plans": { - "type": "object", - "id": {"type": "string"}, - "name": {"type": "string"}, - "description": {"type": "string"}, - "metadata": {"type": "object"}, - "free": {"type": "boolean"}, - "bindable": {"type": "boolean"}, - "binding_rotatable": {"type": "boolean"}, - "plan_updateable": {"type": "boolean"}, - "schemas": {"type": "object"}, - "maximum_polling_duration": {"type": "integer"}, - "maintenance_info": {"type": "object"}, - "required": [ - "id", "name", "description" - ] - }, - "required": [ - "id", "name", "description", "bindable", "version", "plans" - ] - } - } -} def command(cmd, *args, output_type="text"): - status, output = subprocess.getstatusoutput("%s %s" % (cmd, " ".join(args))) # noqa + status, output = subprocess.getstatusoutput("%s %s" % (cmd, " ".join(args))) if output_type == "yaml": return yaml.load(output, Loader=yaml.Loader) elif output_type == "json": @@ -113,13 +25,8 @@ def command(cmd, *args, output_type="text"): return status, output -get_instance_path = lambda instance_id: os.path.join(INSTANCES_PATH, instance_id) # noqa -get_instance_file = lambda instance_id: os.path.join(get_instance_path(instance_id), "instance.json") # noqa -get_chart_path = lambda instance_id: os.path.join(get_instance_path(instance_id), "chart") # noqa -get_plan_path = lambda instance_id: os.path.join(get_instance_path(instance_id), "plan") # noqa - - def helm(instance_id, *args, output_type="text"): + from .database.query import get_instance_path instance_path = get_instance_path(instance_id) new_args = [] new_args.extend(args) @@ -134,145 +41,9 @@ def helm(instance_id, *args, output_type="text"): return command("helm", *new_args, output_type=output_type) -def load_instance_meta(instance_id): - file = get_instance_file(instance_id) - with open(file) as f: - data = json.loads(f.read()) - validate(instance=data, schema=INSTANCE_META_SCHEMA) - return data - - -def dump_instance_meta(instance_id, data): - data["last_modified_time"] = time.time() - file = get_instance_file(instance_id) - validate(instance=data, schema=INSTANCE_META_SCHEMA) - with open(file, "w") as f: - f.write(json.dumps(data, sort_keys=True, indent=2)) - - -def dump_raw_values(instance_id, data): - timestamp = time.time() - instance_path = get_instance_path(instance_id) - file = f"{instance_path}/raw-values-{timestamp}.yaml" - with open(file, "w") as f: - f.write(data) - return file - - -def dump_addon_values(service_id, instance_id): - timestamp = time.time() - instance_path = get_instance_path(instance_id) - file = f"{instance_path}/addon-values-{timestamp}.yaml" - service = _get_addon_meta(service_id) - logger.debug(f"dump_addon_values service: {service}") - if not os.path.exists(f'{CONFIG_PATH}/addon-values'): - return None - with open(file, "w") as fw: - with open(f'{CONFIG_PATH}/addon-values', 'r') as f: - addons_values = yaml.load(f.read(), Loader=yaml.Loader) - logger.debug(f"dump_addon_values addons_values: {addons_values}") - addon_values = addons_values.get(service["name"], {}).\ - get(service["version"], {}) - logger.debug(f"dump_addon_values addon_values: {addon_values}") - if not addon_values: - return None - fw.write(yaml.dump(addon_values)) - return file - - -def load_binding_meta(instance_id): - file = os.path.join(get_instance_path(instance_id), "binding.json") - with open(file, 'r') as f: - data = json.loads(f.read()) - validate(instance=data, schema=INSTANCE_META_SCHEMA) - return data - - -def dump_binding_meta(instance_id, data): - data["last_modified_time"] = time.time() - file = os.path.join(get_instance_path(instance_id), "binding.json") - validate(instance=data, schema=INSTANCE_META_SCHEMA) - with open(file, "w") as f: - f.write(json.dumps(data, sort_keys=True, indent=2)) - - -def load_addons_meta(): - file = os.path.join(ADDONS_PATH, "addons.json") - with open(file, 'r') as f: - data = json.loads(f.read()) - if not data: - return {} - validate(instance=data, schema=INSTANCE_META_SCHEMA) - return data - - -def dump_addons_meta(data): - file = os.path.join(ADDONS_PATH, "addons.json") - validate(instance=data, schema=INSTANCE_META_SCHEMA) - with open(file, "w") as f: - f.write(json.dumps(data, sort_keys=True, indent=2)) - - -def get_addon_path(service_id, plan_id): - service = _get_addon_meta(service_id) - plan = [plan for plan in service['plans'] if plan['id'] == plan_id][0] - plan_name = plan['name'] - service_name_path = f'{service["name"]}-{service["version"]}' - base_path = f"{ADDONS_PATH}/{service_name_path}" - service_path = f'{base_path}/chart/{service["name"]}' - plan_path = f'{base_path}/plans/{plan_name}' - return service_path, plan_path - - -def get_addon_updateable(service_id): - service = _get_addon_meta(service_id) - return service.get('plan_updateable', False) - - -def get_addon_bindable(service_id): - service = _get_addon_meta(service_id) - return service.get('bindable', False) - - -def get_addon_allow_paras(service_id): - service = _get_addon_meta(service_id) - return service.get('allow_parameters', []) - - -def get_addon_archive(service_id): - service = _get_addon_meta(service_id) - return service.get('archive', False) - - -def get_cred_value(ns, source): - if source.get('serviceRef'): - return _get_service_key_value(ns, source['serviceRef']) - if source.get('configMapRef'): - return _get_config_map_key_value(ns, source['configMapRef']) - if source.get('secretKeyRef'): - return _get_secret_key_value(ns, source['secretKeyRef']) - return -1, 'invalid valueFrom' - - -class InstanceLock(object): - - def __init__(self, instance_id): - self.instance_id = instance_id - - def __enter__(self): - self.fileno = open( - os.path.join(INSTANCES_PATH, self.instance_id, "instance.lock"), - "w" - ) - fcntl.flock(self.fileno, fcntl.LOCK_EX) - return self - - def __exit__(self, exc_type, exc_value, traceback): - fcntl.flock(self.fileno, fcntl.LOCK_UN) - - def __del__(self): - if hasattr(self, "fileno"): - fcntl.flock(self.fileno, fcntl.LOCK_UN) +def new_instance_lock(instance_id): + redis = client.Redis.from_url(REDIS_URL) + return redis.lock(instance_id) def verify_parameters(allow_parameters, parameters): @@ -295,15 +66,14 @@ def merge_parameters(parameters): ) -def format_paras_to_helm_args(instance_id, parameters, args): - """ - - """ +def format_params_to_helm_args(instance_id, parameters, args): + """format helm args""" + from .database.savepoint import save_raw_values params = copy.deepcopy(parameters) if params and "rawValues" in params \ and params.get("rawValues", ""): - values = str(base64.b64decode(params["rawValues"]), "utf-8") # noqa - raw_values_file = dump_raw_values(instance_id, values) + values = str(base64.b64decode(params["rawValues"]), "utf-8") + raw_values_file = save_raw_values(instance_id, values) args.extend(["-f", raw_values_file]) params.pop("rawValues") if params: @@ -312,37 +82,6 @@ def format_paras_to_helm_args(instance_id, parameters, args): return args -def _get_addon_meta(service_id): - services = load_addons_meta() - service = [addon for addon in [addons for _, addons in services.items()] - if addon['id'] == service_id][0] - return service - - -def _get_service_key_value(ns, service_ref): - args = [ - "get", "svc", service_ref['name'], "-n", ns, '-o', f"jsonpath=\'{service_ref['jsonpath']}\'", # noqa - ] - return command("kubectl", *args) - - -def _get_config_map_key_value(ns, config_map_ref): - args = [ - "get", "cm", config_map_ref['name'], "-n", ns, '-o', f"jsonpath=\'{config_map_ref['jsonpath']}\'", # noqa - ] - return command("kubectl", *args) - - -def _get_secret_key_value(ns, secret_ref): - args = [ - "get", "secret", secret_ref['name'], "-n", ns, '-o', f"jsonpath=\'{secret_ref['jsonpath']}\'", # noqa - ] - status, output = command("kubectl", *args) - if status == 0: - output = base64.b64decode(output).decode() - return status, output - - def _raw_values_format_keys(raw_values, prefix=''): """ {'a': {'b': 1, 'c': {'d': 2, 'e': 3}}, 'f': 4} diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt index df3a567..b88c2ac 100644 --- a/rootfs/requirements.txt +++ b/rootfs/requirements.txt @@ -3,4 +3,5 @@ gunicorn==22.0.0 openbrokerapi==4.6.0 requests==2.31.0 celery==5.4.0 +redis==5.0.8 jsonschema==4.21.1 \ No newline at end of file diff --git a/rootfs/setup.cfg b/rootfs/setup.cfg new file mode 100644 index 0000000..e000ebb --- /dev/null +++ b/rootfs/setup.cfg @@ -0,0 +1,4 @@ +[flake8] +max-line-length = 99 +exclude = api/migrations,templates,venv +max-complexity = 12 \ No newline at end of file From eecabf79c26dc933e2db1e35f14fd590a9a01eda Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 2 Aug 2024 18:10:39 +0800 Subject: [PATCH 03/28] feat(task): add hooks --- rootfs/helmbroker/database/query.py | 8 +++++ rootfs/helmbroker/database/savepoint.py | 43 ++++++++++++++++--------- rootfs/helmbroker/tasks.py | 10 +++--- rootfs/helmbroker/utils.py | 28 ++++++++++++++++ 4 files changed, 68 insertions(+), 21 deletions(-) diff --git a/rootfs/helmbroker/database/query.py b/rootfs/helmbroker/database/query.py index b87810b..b7aa331 100644 --- a/rootfs/helmbroker/database/query.py +++ b/rootfs/helmbroker/database/query.py @@ -22,6 +22,14 @@ def get_plan_path(instance_id): return os.path.join(get_instance_path(instance_id), "plan") +def get_hooks_path(instance_id): + return os.path.join(get_plan_path(instance_id), "hooks") + + +def get_hooks_result_file(instance_id): + return os.path.join(get_instance_path(instance_id), "hooks-result.json") + + def get_binding_file(instance_id): return os.path.join(get_instance_path(instance_id), "binding.json") diff --git a/rootfs/helmbroker/database/savepoint.py b/rootfs/helmbroker/database/savepoint.py index 2467349..ba97960 100644 --- a/rootfs/helmbroker/database/savepoint.py +++ b/rootfs/helmbroker/database/savepoint.py @@ -1,4 +1,5 @@ import os +import json import logging import yaml import shutil @@ -6,11 +7,31 @@ from ..config import CONFIG_PATH from .query import get_instance_path, get_backups_path, get_addon_meta, get_addon_values_file, \ - get_custom_addon_values_file + get_custom_addon_values_file, get_hooks_result_file logger = logging.getLogger(__name__) +def backup_instance(instance_id): + now = datetime.datetime.now(datetime.timezone.utc) + backup_path = os.path.join(get_backups_path(instance_id), now.isoformat()) + os.makedirs(backup_path, exist_ok=True) + + hooks_result_file = get_hooks_result_file(instance_id) + if os.path.exists(hooks_result_file): + shutil.copy(hooks_result_file, backup_path) + addon_values_file = get_addon_values_file(instance_id) + if os.path.exists(addon_values_file): + shutil.copy(addon_values_file, backup_path) + custom_addon_values_file = get_custom_addon_values_file(instance_id) + if os.path.exists(custom_addon_values_file): + shutil.copy(custom_addon_values_file, backup_path) + + instance_path = get_instance_path(instance_id) + shutil.copytree(os.path.join(instance_path, "plan"), os.path.join(backup_path, "plan")) + shutil.copytree(os.path.join(instance_path, "chart"), os.path.join(backup_path, "chart")) + + def save_raw_values(instance_id, data): file = get_custom_addon_values_file(instance_id) with open(file, "w") as f: @@ -37,18 +58,8 @@ def save_addon_values(service_id, instance_id): return file -def backup_instance(instance_id): - now = datetime.datetime.now(datetime.timezone.utc) - backup_path = os.path.join(get_backups_path(instance_id), now.isoformat()) - os.makedirs(backup_path, exist_ok=True) - - addon_values_file = get_addon_values_file(instance_id) - if os.path.exists(addon_values_file): - shutil.copy(addon_values_file, backup_path) - custom_addon_values_file = get_custom_addon_values_file(instance_id) - if os.path.exists(custom_addon_values_file): - shutil.copy(custom_addon_values_file, backup_path) - - instance_path = get_instance_path(instance_id) - shutil.copytree(os.path.join(instance_path, "plan"), os.path.join(backup_path, "plan")) - shutil.copytree(os.path.join(instance_path, "chart"), os.path.join(backup_path, "chart")) +def save_hooks_result(instance_id, data): + file = get_hooks_result_file(instance_id) + with open(file, "w") as f: + f.write(json.dumps(data, sort_keys=True, indent=2)) + return file diff --git a/rootfs/helmbroker/tasks.py b/rootfs/helmbroker/tasks.py index 22c3e94..fd08b72 100644 --- a/rootfs/helmbroker/tasks.py +++ b/rootfs/helmbroker/tasks.py @@ -7,7 +7,7 @@ UpdateDetails, BindDetails from .celery import app -from .utils import helm, format_params_to_helm_args, new_instance_lock +from .utils import helm, format_params_to_helm_args, new_instance_lock, run_instance_hooks from .database.metadata import save_instance_meta, save_binding_meta, load_instance_meta from .database.savepoint import save_addon_values, backup_instance @@ -18,7 +18,7 @@ @app.task(serializer='pickle') def provision(instance_id: str, details: ProvisionDetails): - with new_instance_lock(instance_id): + with new_instance_lock(instance_id), run_instance_hooks(instance_id, "provision"): backup_instance(instance_id) # create instance.json save_instance_meta(instance_id, { @@ -68,7 +68,7 @@ def provision(instance_id: str, details: ProvisionDetails): @app.task(serializer='pickle') def update(instance_id: str, details: UpdateDetails): - with new_instance_lock(instance_id): + with new_instance_lock(instance_id), run_instance_hooks(instance_id, "update"): backup_instance(instance_id) data = load_instance_meta(instance_id) if details.service_id: @@ -119,7 +119,7 @@ def bind(instance_id: str, details: BindDetails, async_allowed: bool, **kwargs): - with new_instance_lock(instance_id): + with new_instance_lock(instance_id), run_instance_hooks(instance_id, "bind"): backup_instance(instance_id) data = { "binding_id": binding_id, "credentials": {}, @@ -181,7 +181,7 @@ def bind(instance_id: str, @app.task() def deprovision(instance_id: str): - with new_instance_lock(instance_id): + with new_instance_lock(instance_id), run_instance_hooks(instance_id, "deprovision"): backup_instance(instance_id) data = load_instance_meta(instance_id) data["last_operation"]["operation"] = "deprovision" diff --git a/rootfs/helmbroker/utils.py b/rootfs/helmbroker/utils.py index b4271bf..ea664b4 100644 --- a/rootfs/helmbroker/utils.py +++ b/rootfs/helmbroker/utils.py @@ -6,6 +6,7 @@ import copy import logging +from contextlib import contextmanager from redis import client from .config import REDIS_URL @@ -46,6 +47,33 @@ def new_instance_lock(instance_id): return redis.lock(instance_id) +@contextmanager +def run_instance_hooks(instance_id, stage): + if stage not in ["provision", "bind", "unbind", "update", "deprovision"]: + raise ValueError(f"Unknown stage {stage}") + from .database.query import get_hooks_path + from .database.savepoint import save_hooks_result + pre_script_file = os.path.join(get_hooks_path(instance_id), f"pre_{stage}.sh") + post_script_file = os.path.join(get_hooks_path(instance_id), f"post_{stage}.sh") + logger.debug(f"instance hook running: {instance_id}, {instance_id}") + result = [] + try: + if os.path.exists(pre_script_file): + status, output = subprocess.getstatusoutput(pre_script_file) + result.append({"script": pre_script_file, "status": status, "output": output}) + else: + logger.debug(f"skip running {pre_script_file}") + yield + finally: + if os.path.exists(pre_script_file): + status, output = subprocess.getstatusoutput(post_script_file) + result.append({"script": pre_script_file, "status": status, "output": output}) + else: + logger.debug(f"skip running {post_script_file}") + save_hooks_result(instance_id, result) + logger.debug(f"instance hook completed: {instance_id}, {instance_id}") + + def verify_parameters(allow_parameters, parameters): """verify parameters allowed or not""" def merge_parameters(parameters): From cc07840dd74bb93120708353f9cd84e065620a22 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 2 Aug 2024 19:13:43 +0800 Subject: [PATCH 04/28] feat(tasks): change unbind async --- rootfs/helmbroker/broker.py | 15 +++++++-------- rootfs/helmbroker/tasks.py | 13 +++++++++++-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/rootfs/helmbroker/broker.py b/rootfs/helmbroker/broker.py index 561299e..0f82132 100644 --- a/rootfs/helmbroker/broker.py +++ b/rootfs/helmbroker/broker.py @@ -6,7 +6,7 @@ from openbrokerapi.catalog import ServicePlan from openbrokerapi.errors import ErrInstanceAlreadyExists, ErrAsyncRequired, \ ErrBindingAlreadyExists, ErrBadRequest, ErrInstanceDoesNotExist, \ - ServiceException, ErrBindingDoesNotExist + ServiceException from openbrokerapi.service_broker import ServiceBroker, Service, \ ProvisionDetails, ProvisionedServiceSpec, ProvisionState, GetBindingSpec, \ BindDetails, Binding, BindState, UnbindDetails, UnbindSpec, \ @@ -19,7 +19,7 @@ get_addon_updateable, get_addon_bindable, get_addon_allow_params, \ get_addon_archive, get_binding_file, get_instance_file from .database.metadata import load_instance_meta, load_binding_meta, load_addons_meta -from .tasks import provision, bind, deprovision, update +from .tasks import provision, bind, deprovision, update, unbind logger = logging.getLogger(__name__) @@ -112,10 +112,9 @@ def unbind(self, async_allowed: bool, **kwargs ) -> UnbindSpec: - binding_file = get_binding_file(instance_id) - if os.path.exists(binding_file): - os.remove(binding_file) - return UnbindSpec(is_async=False) + logger.debug(f"unbind instance {instance_id}") + unbind.delay(instance_id) + return UnbindSpec(is_async=True) def update(self, instance_id: str, @@ -179,7 +178,7 @@ def last_operation(self, OperationState(data["last_operation"]["state"]), data["last_operation"]["description"] ) - raise ErrInstanceDoesNotExist() + raise LastOperation(OperationState.IN_PROGRESS) def last_binding_operation(self, instance_id: str, @@ -193,4 +192,4 @@ def last_binding_operation(self, OperationState(data["last_operation"]["state"]), data["last_operation"]["description"] ) - raise ErrBindingDoesNotExist() + return LastOperation(OperationState.SUCCEEDED) diff --git a/rootfs/helmbroker/tasks.py b/rootfs/helmbroker/tasks.py index fd08b72..1b6066d 100644 --- a/rootfs/helmbroker/tasks.py +++ b/rootfs/helmbroker/tasks.py @@ -11,7 +11,7 @@ from .database.metadata import save_instance_meta, save_binding_meta, load_instance_meta from .database.savepoint import save_addon_values, backup_instance -from .database.query import get_plan_path, get_chart_path, get_cred_value +from .database.query import get_plan_path, get_chart_path, get_cred_value, get_binding_file logger = logging.getLogger(__name__) @@ -179,7 +179,16 @@ def bind(instance_id: str, save_binding_meta(instance_id, data) -@app.task() +@app.task(serializer='pickle') +def unbind(instance_id): + with new_instance_lock(instance_id), run_instance_hooks(instance_id, "deprovision"): + backup_instance(instance_id) + binding_file = get_binding_file(instance_id) + if os.path.exists(binding_file): + os.remove(binding_file) + + +@app.task(serializer='pickle') def deprovision(instance_id: str): with new_instance_lock(instance_id), run_instance_hooks(instance_id, "deprovision"): backup_instance(instance_id) From 60d66c8eafb28ca515beb1f85e5b44dd2985fb79 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Mon, 5 Aug 2024 17:37:37 +0800 Subject: [PATCH 05/28] chore(hooks): always run save_hooks_result --- rootfs/helmbroker/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rootfs/helmbroker/utils.py b/rootfs/helmbroker/utils.py index ea664b4..6114f4a 100644 --- a/rootfs/helmbroker/utils.py +++ b/rootfs/helmbroker/utils.py @@ -70,7 +70,7 @@ def run_instance_hooks(instance_id, stage): result.append({"script": pre_script_file, "status": status, "output": output}) else: logger.debug(f"skip running {post_script_file}") - save_hooks_result(instance_id, result) + save_hooks_result(instance_id, result) logger.debug(f"instance hook completed: {instance_id}, {instance_id}") From 658ba12c2c34f363ab88210f026600701ded4121 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Mon, 5 Aug 2024 20:01:29 +0800 Subject: [PATCH 06/28] chore(broker): add check install_hooks status --- rootfs/helmbroker/broker.py | 4 +- rootfs/helmbroker/tasks.py | 121 +++++++++++++++++++++++------------- rootfs/helmbroker/utils.py | 5 +- 3 files changed, 84 insertions(+), 46 deletions(-) diff --git a/rootfs/helmbroker/broker.py b/rootfs/helmbroker/broker.py index 0f82132..8b38269 100644 --- a/rootfs/helmbroker/broker.py +++ b/rootfs/helmbroker/broker.py @@ -178,7 +178,7 @@ def last_operation(self, OperationState(data["last_operation"]["state"]), data["last_operation"]["description"] ) - raise LastOperation(OperationState.IN_PROGRESS) + return LastOperation(OperationState.IN_PROGRESS) def last_binding_operation(self, instance_id: str, @@ -192,4 +192,4 @@ def last_binding_operation(self, OperationState(data["last_operation"]["state"]), data["last_operation"]["description"] ) - return LastOperation(OperationState.SUCCEEDED) + return LastOperation(OperationState.IN_PROGRESS) diff --git a/rootfs/helmbroker/tasks.py b/rootfs/helmbroker/tasks.py index 1b6066d..9cfda51 100644 --- a/rootfs/helmbroker/tasks.py +++ b/rootfs/helmbroker/tasks.py @@ -9,7 +9,8 @@ from .celery import app from .utils import helm, format_params_to_helm_args, new_instance_lock, run_instance_hooks -from .database.metadata import save_instance_meta, save_binding_meta, load_instance_meta +from .database.metadata import save_instance_meta, save_binding_meta, load_instance_meta, \ + load_binding_meta from .database.savepoint import save_addon_values, backup_instance from .database.query import get_plan_path, get_chart_path, get_cred_value, get_binding_file @@ -18,22 +19,30 @@ @app.task(serializer='pickle') def provision(instance_id: str, details: ProvisionDetails): - with new_instance_lock(instance_id), run_instance_hooks(instance_id, "provision"): + with ( + new_instance_lock(instance_id), + run_instance_hooks(instance_id, "provision") as (status, output) + ): backup_instance(instance_id) # create instance.json - save_instance_meta(instance_id, { - "id": instance_id, + data = { + "id": instance_id, "last_operation": {}, "details": { "service_id": details.service_id, "plan_id": details.plan_id, "context": details.context, "parameters": details.parameters if details.parameters else {}, }, - "last_operation": { - "state": OperationState.IN_PROGRESS.value, "operation": "provision", - "description": ("provision %s in progress at %s" % (instance_id, time.time())) - } - }) - + } + if status != 0: + data["last_operation"]["state"] = OperationState.FAILED.value + data["last_operation"]["description"] = f"provision {instance_id} error: {output}" + save_instance_meta(instance_id, data) + return + data["last_operation"]["state"] = OperationState.IN_PROGRESS.value + data["last_operation"]["operation"] = "provision" + data["last_operation"]["description"] = ( + f"provision {instance_id} in progress at {time.time()}") + save_instance_meta(instance_id, data) chart_path = get_chart_path(instance_id) bind_yaml = f'{chart_path}/templates/bind.yaml' if os.path.exists(bind_yaml): @@ -56,19 +65,22 @@ def provision(instance_id: str, details: ProvisionDetails): args = format_params_to_helm_args(instance_id, details.parameters, args) logger.debug(f"helm install args:{args}") status, output = helm(instance_id, *args) - data = load_instance_meta(instance_id) if status != 0: data["last_operation"]["state"] = OperationState.FAILED.value - data["last_operation"]["description"] = "provision error:\n%s" % output + data["last_operation"]["description"] = f"provision {instance_id} error: {output}" else: data["last_operation"]["state"] = OperationState.SUCCEEDED.value - data["last_operation"]["description"] = "provision succeeded at %s" % time.time() + data["last_operation"]["description"] = ( + f"provision {instance_id} succeeded at {time.time()}") save_instance_meta(instance_id, data) @app.task(serializer='pickle') def update(instance_id: str, details: UpdateDetails): - with new_instance_lock(instance_id), run_instance_hooks(instance_id, "update"): + with ( + new_instance_lock(instance_id), + run_instance_hooks(instance_id, "update") as (status, output) + ): backup_instance(instance_id) data = load_instance_meta(instance_id) if details.service_id: @@ -82,9 +94,14 @@ def update(instance_id: str, details: UpdateDetails): params.update(details.parameters) # remove the key which value is null data['details']['parameters'] = {k: v for k, v in params.items() if v != ""} + if status != 0: + data["last_operation"]["state"] = OperationState.FAILED.value + data["last_operation"]["description"] = f"update {instance_id} failed: {output}" + save_instance_meta(instance_id, data) + return data['last_operation']["state"] = OperationState.IN_PROGRESS.value - data['last_operation']["description"] = "update %s in progress at %s" % ( - instance_id, time.time()) + data['last_operation']["description"] = ( + f"update {instance_id} in progress at {time.time()}") save_instance_meta(instance_id, data) chart_path = get_chart_path(instance_id) values_file = os.path.join(get_plan_path(instance_id), "values.yaml") @@ -105,11 +122,11 @@ def update(instance_id: str, details: UpdateDetails): status, output = helm(instance_id, *args) if status != 0: data["last_operation"]["state"] = OperationState.FAILED.value - data["last_operation"]["description"] = "update %s failed: %s" % (instance_id, output) + data["last_operation"]["description"] = f"update {time.time()} failed: {output}" else: data["last_operation"]["state"] = OperationState.SUCCEEDED.value data["last_operation"]["description"] = ( - "update %s succeeded at %s" % (instance_id, time.time())) + f"update {instance_id} succeeded at {time.time()}") save_instance_meta(instance_id, data) @@ -119,15 +136,17 @@ def bind(instance_id: str, details: BindDetails, async_allowed: bool, **kwargs): - with new_instance_lock(instance_id), run_instance_hooks(instance_id, "bind"): + with ( + new_instance_lock(instance_id), + run_instance_hooks(instance_id, "bind") as (status, output) + ): backup_instance(instance_id) - data = { - "binding_id": binding_id, "credentials": {}, - "last_operation": { - "state": OperationState.IN_PROGRESS.value, - "description": "binding %s in progress at %s" % (binding_id, time.time()) - } - } + data = {"binding_id": binding_id, "credentials": {}, "last_operation": {}} + if status != 0: + data["last_operation"]["state"] = OperationState.FAILED.value + data["last_operation"]["description"] = f"binding {instance_id} failed: {output}" + save_binding_meta(instance_id, data) + return save_binding_meta(instance_id, data) chart_path = get_chart_path(instance_id) values_file = os.path.join(get_plan_path(instance_id), "values.yaml") @@ -145,9 +164,7 @@ def bind(instance_id: str, status, templates = helm(instance_id, *args) # output: templates.yaml if status != 0: data["last_operation"]["state"] = OperationState.FAILED.value - data["last_operation"]["description"] = "binding %s failed: %s" % ( - instance_id, templates) - + data["last_operation"]["description"] = f"binding {instance_id} failed: {templates}" credential_template = yaml.load(templates.split('bind.yaml')[1], Loader=yaml.Loader) success_flag = True errors = [] @@ -164,15 +181,13 @@ def bind(instance_id: str, else: data['credentials'][_['name']] = val if success_flag: - data['last_operation'] = { - 'state': OperationState.SUCCEEDED.value, - 'description': "binding %s succeeded at %s" % (instance_id, time.time()) - } + data['last_operation']['state'] = OperationState.SUCCEEDED.value + data['last_operation']['description'] = ( + f"binding {instance_id} succeeded at {time.time()}") else: - data['last_operation'] = { - 'state': OperationState.FAILED.value, - 'description': "binding %s failed: %s" % (instance_id, ','.join(errors)) - } + data['last_operation']['state'] = OperationState.FAILED.value + data['last_operation']['description'] = ( + f"binding {instance_id} failed: {','.join(errors)}") bind_yaml = f'{chart_path}/templates/bind.yaml' if os.path.exists(bind_yaml): os.remove(bind_yaml) @@ -181,22 +196,43 @@ def bind(instance_id: str, @app.task(serializer='pickle') def unbind(instance_id): - with new_instance_lock(instance_id), run_instance_hooks(instance_id, "deprovision"): + with ( + new_instance_lock(instance_id), + run_instance_hooks(instance_id, "deprovision") as (status, output) + ): backup_instance(instance_id) + data = load_binding_meta(instance_id) + if status != 0: + data['last_operation']['state'] = OperationState.FAILED.value + data['last_operation']['description'] = f"unbind {instance_id} failed: {output}" + save_binding_meta(instance_id, data) + return binding_file = get_binding_file(instance_id) if os.path.exists(binding_file): os.remove(binding_file) + data['last_operation']['state'] = OperationState.SUCCEEDED.value + data['last_operation']['description'] = f"unbind {instance_id} succeeded at {time.time()}" + save_binding_meta(instance_id, data) @app.task(serializer='pickle') def deprovision(instance_id: str): - with new_instance_lock(instance_id), run_instance_hooks(instance_id, "deprovision"): + with ( + new_instance_lock(instance_id), + run_instance_hooks(instance_id, "deprovision") as (status, output) + ): backup_instance(instance_id) data = load_instance_meta(instance_id) + if status != 0: + data["last_operation"]["operation"] = "deprovision" + data["last_operation"]["state"] = OperationState.FAILED.value + data["last_operation"]["description"] = f"deprovision {instance_id} failed: {output}" + save_instance_meta(instance_id, data) + return data["last_operation"]["operation"] = "deprovision" data["last_operation"]["state"] = OperationState.IN_PROGRESS.value data["last_operation"]["description"] = ( - "deprovision %s in progress at %s" % (instance_id, time.time())) + f"deprovision {instance_id} in progress at {time.time()}") save_instance_meta(instance_id, data) args = [ "uninstall", data["details"]["context"]["instance_name"], @@ -206,8 +242,9 @@ def deprovision(instance_id: str): status, output = helm(instance_id, *args) if status != 0: data["last_operation"]["state"] = OperationState.FAILED.value - data["last_operation"]["description"] = "deprovision error:\n%s" % output + data["last_operation"]["description"] = f"deprovision {instance_id} failed: {output}" else: data["last_operation"]["state"] = OperationState.SUCCEEDED.value - data["last_operation"]["description"] = "deprovision succeeded at %s" % time.time() + data["last_operation"]["description"] = ( + f"deprovision {instance_id} succeeded at {time.time()}") save_instance_meta(instance_id, data) diff --git a/rootfs/helmbroker/utils.py b/rootfs/helmbroker/utils.py index 6114f4a..ff19967 100644 --- a/rootfs/helmbroker/utils.py +++ b/rootfs/helmbroker/utils.py @@ -62,8 +62,9 @@ def run_instance_hooks(instance_id, stage): status, output = subprocess.getstatusoutput(pre_script_file) result.append({"script": pre_script_file, "status": status, "output": output}) else: - logger.debug(f"skip running {pre_script_file}") - yield + status, output = 0, f"skip running {pre_script_file}" + logger.debug(output) + yield status, output finally: if os.path.exists(pre_script_file): status, output = subprocess.getstatusoutput(post_script_file) From 1b56efaccc03b87566d0732f8a54ad3bd90045c0 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 6 Aug 2024 10:04:48 +0800 Subject: [PATCH 07/28] chore(helmbroker): bump new version --- .woodpecker/test-linux.yml | 3 ++- Makefile | 2 +- rootfs/Dockerfile | 6 +++--- rootfs/Dockerfile.test | 10 ++++++---- rootfs/bin/test-unit | 2 +- rootfs/bin/upload-coverage | 6 +++++- rootfs/dev_requirements.txt | 9 +++------ rootfs/helmbroker/tasks.py | 6 +++--- 8 files changed, 24 insertions(+), 20 deletions(-) diff --git a/.woodpecker/test-linux.yml b/.woodpecker/test-linux.yml index d47ccc5..16a01b5 100644 --- a/.woodpecker/test-linux.yml +++ b/.woodpecker/test-linux.yml @@ -11,10 +11,11 @@ steps: - name: test-linux image: bash commands: - - make test + - make test upload-coverage secrets: - codename - dev_registry + - codecov_token when: event: - push diff --git a/Makefile b/Makefile index 26df599..fed80c8 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,6 @@ test-integration: upload-coverage: $(eval CI_ENV := $(shell curl -s https://codecov.io/env | bash)) - podman run --rm ${CI_ENV} -v ${CURDIR}:/tmp/test -w /tmp/test/rootfs ${IMAGE}.test /tmp/test/rootfs/bin/upload-coverage + podman run --rm ${CI_ENV} -v ${CURDIR}:/tmp/test -w /tmp/test/rootfs -e CODECOV_TOKEN=${CODECOV_TOKEN} ${IMAGE}.test /tmp/test/rootfs/bin/upload-coverage .PHONY: check-kubectl check-podman build podman-build podman-build-test deploy clean commit-hook full-clean test test-style test-unit test-functional test-integration upload-coverage diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 806f7c7..e97938a 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -4,9 +4,9 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/workspace \ - PYTHON_VERSION="3.11" \ - HELM_VERSION="3.12.1" \ - KUBECTL_VERSION="1.27.3" + PYTHON_VERSION="3.12" \ + HELM_VERSION="3.15.3" \ + KUBECTL_VERSION="1.30.3" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test index 8cbb6e6..b5c1d34 100644 --- a/rootfs/Dockerfile.test +++ b/rootfs/Dockerfile.test @@ -4,9 +4,9 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/workspace \ - PYTHON_VERSION="3.11" \ - HELM_VERSION="3.12.1" \ - KUBECTL_VERSION="1.27.3" + PYTHON_VERSION="3.12" \ + HELM_VERSION="3.15.3" \ + KUBECTL_VERSION="1.30.3" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} @@ -15,7 +15,9 @@ COPY . ${DRYCC_HOME_DIR} WORKDIR ${DRYCC_HOME_DIR} RUN buildDeps='musl-dev'; \ - install-packages ${buildDeps} openssl ca-certificates \ + install-packages ${buildDeps} git openssl ca-certificates \ + && curl -SsL https://cli.codecov.io/latest/$([ $(dpkg --print-architecture) == "arm64" ] && echo linux-arm64 || echo linux)/codecov -o /usr/local/bin/codecov \ + && chmod +x /usr/local/bin/codecov \ && install-stack python ${PYTHON_VERSION} \ && install-stack helm ${HELM_VERSION} \ && install-stack kubectl ${KUBECTL_VERSION} && . init-stack \ diff --git a/rootfs/bin/test-unit b/rootfs/bin/test-unit index ef9755f..1d1d54e 100755 --- a/rootfs/bin/test-unit +++ b/rootfs/bin/test-unit @@ -3,4 +3,4 @@ # This script is designed to be run inside the container # -python -m unittest tests/test_*.py \ No newline at end of file +coverage run -m unittest tests/test_*.py diff --git a/rootfs/bin/upload-coverage b/rootfs/bin/upload-coverage index 53433f3..492452b 100755 --- a/rootfs/bin/upload-coverage +++ b/rootfs/bin/upload-coverage @@ -6,4 +6,8 @@ # fail hard and fast even on pipelines set -eou pipefail -codecov --required +coverage report -m > coverage.txt + +if [[ -n $CODECOV_TOKEN ]]; then + codecov upload-process --plugin=noop -t "$CODECOV_TOKEN" +fi diff --git a/rootfs/dev_requirements.txt b/rootfs/dev_requirements.txt index b46d2dc..ca19f67 100644 --- a/rootfs/dev_requirements.txt +++ b/rootfs/dev_requirements.txt @@ -1,13 +1,10 @@ # test module # test # Run "make test-unit" for the % of code exercised during tests -coverage==7.2.7 +coverage==7.6.1 # Run "make test-style" to check python syntax and style -flake8==6.0.0 - -# code coverage report at https://codecov.io/github/drycc/controller -codecov==2.1.13 +flake8==7.1.1 # mock out python-requests, mostly k8s -requests-mock==1.11.0 +requests-mock==1.12.1 diff --git a/rootfs/helmbroker/tasks.py b/rootfs/helmbroker/tasks.py index 9cfda51..baf7aae 100644 --- a/rootfs/helmbroker/tasks.py +++ b/rootfs/helmbroker/tasks.py @@ -61,9 +61,9 @@ def provision(instance_id: str, details: ProvisionDetails): if addon_values_file: args.insert(9, "-f") args.insert(10, addon_values_file) - logger.debug(f"helm install parameters :{details.parameters}") + logger.debug(f"helm install parameters: {details.parameters}") args = format_params_to_helm_args(instance_id, details.parameters, args) - logger.debug(f"helm install args:{args}") + logger.debug(f"helm install args: {args}") status, output = helm(instance_id, *args) if status != 0: data["last_operation"]["state"] = OperationState.FAILED.value @@ -118,7 +118,7 @@ def update(instance_id: str, details: UpdateDetails): params = data['details']['parameters'] logger.debug(f"helm upgrade parameters: {params}") args = format_params_to_helm_args(instance_id, params, args) - logger.debug(f"helm upgrade args:{args}") + logger.debug(f"helm upgrade args: {args}") status, output = helm(instance_id, *args) if status != 0: data["last_operation"]["state"] = OperationState.FAILED.value From 880aed0384a9c2e6fae7c058f4fed7b0dab6f9bb Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Thu, 12 Sep 2024 17:37:06 +0800 Subject: [PATCH 08/28] chore(celery): use quorum queye --- charts/helmbroker/templates/_helpers.tpl | 2 +- rootfs/helmbroker/celery.py | 30 ++++++++---------------- rootfs/requirements.txt | 4 ++-- 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/charts/helmbroker/templates/_helpers.tpl b/charts/helmbroker/templates/_helpers.tpl index 39282a7..cc0c838 100644 --- a/charts/helmbroker/templates/_helpers.tpl +++ b/charts/helmbroker/templates/_helpers.tpl @@ -22,7 +22,7 @@ env: name: rabbitmq-creds key: password - name: "HELMBROKER_RABBITMQ_URL" - value: "amqp://$(HELMBROKER_RABBITMQ_USERNAME):$(HELMBROKER_RABBITMQ_PASSWORD)@drycc-rabbitmq.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:5672/drycc" + value: "amqp://$(HELMBROKER_RABBITMQ_USERNAME):$(HELMBROKER_RABBITMQ_PASSWORD)@drycc-rabbitmq.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:5672/helmbroker" {{- end }} {{- if (.Values.redisUrl) }} - name: HELMBROKER_REDIS_URL diff --git a/rootfs/helmbroker/celery.py b/rootfs/helmbroker/celery.py index e9699d2..67f810f 100644 --- a/rootfs/helmbroker/celery.py +++ b/rootfs/helmbroker/celery.py @@ -33,43 +33,33 @@ class Config(object): task_routes={ 'helmbroker.tasks.provision': { 'queue': 'helmbroker.high', - 'exchange': 'helmbroker.priority', - 'routing_key': 'helmbroker.priority.high', + 'exchange': 'helmbroker.priority', 'routing_key': 'helmbroker.priority.high', }, 'helmbroker.tasks.update': { 'queue': 'helmbroker.high', - 'exchange': 'helmbroker.priority', - 'routing_key': 'helmbroker.priority.high', + 'exchange': 'helmbroker.priority', 'routing_key': 'helmbroker.priority.high', }, 'helmbroker.tasks.bind': { 'queue': 'helmbroker.high', - 'exchange': 'helmbroker.priority', - 'routing_key': 'helmbroker.priority.high', + 'exchange': 'helmbroker.priority', 'routing_key': 'helmbroker.priority.high', }, 'helmbroker.tasks.deprovision': { 'queue': 'helmbroker.middle', - 'exchange': 'helmbroker.priority', - 'routing_key': 'helmbroker.priority.middle', + 'exchange': 'helmbroker.priority', 'routing_key': 'helmbroker.priority.middle', }, }, task_queues=( Queue( - 'helmbroker.low', - exchange=Exchange('helmbroker.priority', type="direct"), - routing_key='helmbroker.priority.low', - queue_arguments={'x-max-priority': 16}, + 'helmbroker.low', exchange=Exchange('helmbroker.priority', type="direct"), + routing_key='helmbroker.priority.low', queue_arguments={'x-queue-type': 'quorum'}, ), Queue( - 'helmbroker.high', - exchange=Exchange('helmbroker.priority', type="direct"), - routing_key='helmbroker.priority.high', - queue_arguments={'x-max-priority': 64}, + 'helmbroker.high', exchange=Exchange('helmbroker.priority', type="direct"), + routing_key='helmbroker.priority.high', queue_arguments={'x-queue-type': 'quorum'}, ), Queue( - 'helmbroker.middle', - exchange=Exchange('helmbroker.priority', type="direct"), - routing_key='helmbroker.priority.middle', - queue_arguments={'x-max-priority': 32}, + 'helmbroker.middle', exchange=Exchange('helmbroker.priority', type="direct"), + routing_key='helmbroker.priority.middle', queue_arguments={'x-queue-type': 'quorum'}, ), ), ) diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt index b88c2ac..c889eb3 100644 --- a/rootfs/requirements.txt +++ b/rootfs/requirements.txt @@ -2,6 +2,6 @@ PyYAML==6.0.1 gunicorn==22.0.0 openbrokerapi==4.6.0 requests==2.31.0 -celery==5.4.0 +celery==5.5.0b3 redis==5.0.8 -jsonschema==4.21.1 \ No newline at end of file +jsonschema==4.21.1 From fbcd19a66c044feb1b5730a57e06641edcb70e90 Mon Sep 17 00:00:00 2001 From: lijianguo Date: Mon, 28 Oct 2024 17:55:21 +0800 Subject: [PATCH 09/28] chore(helmbroker): yaml load all templates --- rootfs/helmbroker/tasks.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rootfs/helmbroker/tasks.py b/rootfs/helmbroker/tasks.py index baf7aae..b51cf7a 100644 --- a/rootfs/helmbroker/tasks.py +++ b/rootfs/helmbroker/tasks.py @@ -165,10 +165,14 @@ def bind(instance_id: str, if status != 0: data["last_operation"]["state"] = OperationState.FAILED.value data["last_operation"]["description"] = f"binding {instance_id} failed: {templates}" - credential_template = yaml.load(templates.split('bind.yaml')[1], Loader=yaml.Loader) + credential_template = {} + templates = yaml.load_all(templates, Loader=yaml.SafeLoader) + credential_template = next( + (item for item in templates if isinstance(item, dict) and "credential" in item), {} + ) success_flag = True errors = [] - for _ in credential_template['credential']: + for _ in credential_template.get('credential', {}): if _.get('valueFrom'): status, val = get_cred_value(details.context["namespace"], _['valueFrom']) elif _.get('value'): From 3dd6d176b44e0031f0a26728b744ebbcd6434070 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 13 Nov 2024 08:25:57 +0800 Subject: [PATCH 10/28] chore(tasks): use reset-then-reuse-values replace reuse-values --- rootfs/helmbroker/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rootfs/helmbroker/tasks.py b/rootfs/helmbroker/tasks.py index b51cf7a..8f19af7 100644 --- a/rootfs/helmbroker/tasks.py +++ b/rootfs/helmbroker/tasks.py @@ -108,7 +108,7 @@ def update(instance_id: str, details: UpdateDetails): args = [ "upgrade", details.context["instance_name"], chart_path, "--namespace", details.context["namespace"], "--create-namespace", - "--wait", "--timeout", "25m0s", "--reuse-values", "-f", values_file, + "--wait", "--timeout", "25m0s", "--reset-then-reuse-values", "-f", values_file, "--set", f"fullnameOverride=helmbroker-{details.context['instance_name']}" ] addon_values_file = save_addon_values(details.service_id, instance_id) From 24d4d571509ac4cb748318b5d7f31cc42276392b Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Thu, 14 Nov 2024 09:16:11 +0800 Subject: [PATCH 11/28] chore(python): upgrade requirements version --- rootfs/requirements.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt index c889eb3..1141fae 100644 --- a/rootfs/requirements.txt +++ b/rootfs/requirements.txt @@ -1,7 +1,7 @@ -PyYAML==6.0.1 -gunicorn==22.0.0 -openbrokerapi==4.6.0 -requests==2.31.0 -celery==5.5.0b3 -redis==5.0.8 -jsonschema==4.21.1 +PyYAML==6.0.2 +gunicorn==23.0.0 +openbrokerapi==4.7.1 +requests==2.32.2 +celery==5.5.0rc1 +redis==5.2.0 +jsonschema==4.23.0 From 82c6ca41b6d07d036d4fba699968c8ec4ee031e3 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Sun, 24 Nov 2024 20:17:33 +0800 Subject: [PATCH 12/28] feat(valkey): add valkey sentinel support --- .woodpecker/chart.yaml | 1 + charts/helmbroker/Chart.yaml | 7 +-- charts/helmbroker/templates/_helpers.tpl | 37 ++++---------- .../templates/helmbroker-deployment.yaml | 2 +- .../templates/helmbroker-secret-creds.yaml | 10 ++++ charts/helmbroker/values.yaml | 32 ++++-------- rootfs/helmbroker/celery.py | 49 +++++++++++++------ rootfs/helmbroker/config.py | 2 +- rootfs/helmbroker/database/fetch.py | 2 +- rootfs/helmbroker/database/metadata.py | 31 ++++++------ rootfs/helmbroker/utils.py | 26 +++++++--- rootfs/requirements.txt | 2 +- 12 files changed, 104 insertions(+), 97 deletions(-) create mode 100644 charts/helmbroker/templates/helmbroker-secret-creds.yaml diff --git a/.woodpecker/chart.yaml b/.woodpecker/chart.yaml index 214841c..67df5de 100644 --- a/.woodpecker/chart.yaml +++ b/.woodpecker/chart.yaml @@ -11,6 +11,7 @@ steps: - export APP_VERSION=$([ -z $CI_COMMIT_TAG ] && echo $CI_COMMIT_SHA || echo $VERSION) - export CHART_VERSION=$([ -z $CI_COMMIT_TAG ] && echo 1.0.0 || echo $VERSION) - sed -i "s/imageTag:\ \"canary\"/imageTag:\ $IMAGE_TAG/g" charts/$${CI_REPO_NAME}/values.yaml + - sed -i s#{{repository}}#oci://$DRYCC_REGISTRY/$([ -z $CI_COMMIT_TAG ] && echo charts-testing || echo charts)#g charts/$${CI_REPO_NAME}/Chart.yaml - helm package -u charts/$${CI_REPO_NAME} --version $CHART_VERSION --app-version $APP_VERSION - echo $CONTAINER_PASSWORD | helm registry login $DRYCC_REGISTRY -u $CONTAINER_USERNAME --password-stdin - helm push $${CI_REPO_NAME}-$CHART_VERSION.tgz oci://$DRYCC_REGISTRY/$([ -z $CI_COMMIT_TAG ] && echo charts-testing || echo charts) diff --git a/charts/helmbroker/Chart.yaml b/charts/helmbroker/Chart.yaml index da3007f..6e22e3a 100644 --- a/charts/helmbroker/Chart.yaml +++ b/charts/helmbroker/Chart.yaml @@ -6,11 +6,8 @@ dependencies: - name: common repository: oci://registry.drycc.cc/charts version: ~1.1.2 - - name: redis - repository: oci://registry.drycc.cc/charts - version: x.x.x - - name: rabbitmq - repository: oci://registry.drycc.cc/charts + - name: valkey + repository: {{repository}} version: x.x.x description: Drycc Workflow helmbroker. maintainers: diff --git a/charts/helmbroker/templates/_helpers.tpl b/charts/helmbroker/templates/_helpers.tpl index cc0c838..4702bc6 100644 --- a/charts/helmbroker/templates/_helpers.tpl +++ b/charts/helmbroker/templates/_helpers.tpl @@ -7,39 +7,20 @@ env: value: {{ if .Values.username | default "" | ne "" }}{{ .Values.username }}{{ else }}{{ randAlphaNum 32 }}{{ end }} - name: HELMBROKER_PASSWORD value: {{ if .Values.password | default "" | ne "" }}{{ .Values.password }}{{ else }}{{ randAlphaNum 32 }}{{ end }} -{{- if (.Values.rabbitmqUrl) }} -- name: HELMBROKER_RABBITMQ_URL - value: {{ .Values.rabbitmqUrl }} -{{- else if eq .Values.global.rabbitmqLocation "on-cluster" }} -- name: "HELMBROKER_RABBITMQ_USERNAME" +{{- if (.Values.valkeyUrl) }} +- name: HELMBROKER_VALKEY_URL valueFrom: secretKeyRef: - name: rabbitmq-creds - key: username -- name: "HELMBROKER_RABBITMQ_PASSWORD" + name: helmbroker-creds + key: valkey-url +{{- else if eq .Values.global.valkeyLocation "on-cluster" }} +- name: VALKEY_PASSWORD valueFrom: secretKeyRef: - name: rabbitmq-creds + name: valkey-creds key: password -- name: "HELMBROKER_RABBITMQ_URL" - value: "amqp://$(HELMBROKER_RABBITMQ_USERNAME):$(HELMBROKER_RABBITMQ_PASSWORD)@drycc-rabbitmq.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:5672/helmbroker" -{{- end }} -{{- if (.Values.redisUrl) }} -- name: HELMBROKER_REDIS_URL - value: {{ .Values.redisUrl }} -{{- else if eq .Values.global.redisLocation "on-cluster" }} -- name: "HELMBROKER_REDIS_ADDRS" - valueFrom: - secretKeyRef: - name: redis-creds - key: addrs -- name: "HELMBROKER_REDIS_PASSWORD" - valueFrom: - secretKeyRef: - name: redis-creds - key: password -- name: "HELMBROKER_REDIS_URL" - value: "redis://:$(HELMBROKER_REDIS_PASSWORD)@$(HELMBROKER_REDIS_ADDRS)/0" +- name: HELMBROKER_VALKEY_URL + value: "redis://:$(VALKEY_PASSWORD)@drycc-valkey.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:26379/0?master_set=drycc" {{- end }} {{- range $key, $value := .Values.environment }} - name: {{ $key }} diff --git a/charts/helmbroker/templates/helmbroker-deployment.yaml b/charts/helmbroker/templates/helmbroker-deployment.yaml index 7a711ed..6ad1385 100644 --- a/charts/helmbroker/templates/helmbroker-deployment.yaml +++ b/charts/helmbroker/templates/helmbroker-deployment.yaml @@ -34,7 +34,7 @@ spec: - netcat - -v - -u - - $(HELMBROKER_REDIS_URL),$(HELMBROKER_RABBITMQ_URL) + - $(HELMBROKER_VALKEY_URL) {{- include "helmbroker.envs" . | indent 10 }} - name: drycc-helmbroker-fetch image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/helmbroker:{{.Values.imageTag}} diff --git a/charts/helmbroker/templates/helmbroker-secret-creds.yaml b/charts/helmbroker/templates/helmbroker-secret-creds.yaml new file mode 100644 index 0000000..29b1bc3 --- /dev/null +++ b/charts/helmbroker/templates/helmbroker-secret-creds.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: helmbroker-creds + labels: + heritage: drycc +data: + {{- if (.Values.valkeyUrl) }} + valkey-url: {{ .Values.valkeyUrl | b64enc }} + {{- end }} \ No newline at end of file diff --git a/charts/helmbroker/values.yaml b/charts/helmbroker/values.yaml index d8c5375..60915f7 100644 --- a/charts/helmbroker/values.yaml +++ b/charts/helmbroker/values.yaml @@ -23,8 +23,8 @@ diagnosticMode: ## config the helm-broker repositories repositories: -- name: drycc-helm-broker - url: https://github.com/drycc/addons/releases/download/latest/index.yaml + - name: drycc-helm-broker + url: https://github.com/drycc/addons/releases/download/latest/index.yaml celeryReplicas: 1 @@ -33,11 +33,8 @@ celeryReplicas: 1 username: admin password: admin -# Configuring this will no longer use the built-in redis component -redisUrl: "" - -# Configuring this will no longer use the built-in rabbitmq component -rabbitmqUrl: "" +# Configuring this will no longer use the built-in valkey component +valkeyUrl: "" # Any custom controller environment variables # can be specified as key-value pairs under environment @@ -51,7 +48,7 @@ api: key: "drycc.cc/node" type: "soft" values: - - "true" + - "true" podAffinityPreset: type: "" extraMatchLabels: @@ -66,7 +63,7 @@ celery: key: "drycc.cc/node" type: "soft" values: - - "true" + - "true" podAffinityPreset: type: "" extraMatchLabels: @@ -76,10 +73,6 @@ celery: extraMatchLabels: app: "drycc-helmbroker-celery" -# drycc redis replicas must always be set to 1 -redis: - replicas: 1 - # Default override of addon values addonValues: {} @@ -91,16 +84,11 @@ persistence: volumeName: "" global: - # Set the location of Workflow's redis instance - # Valid values are: - # - on-cluster: Run Redis within the Kubernetes cluster - # - off-cluster: Run Redis outside the Kubernetes cluster (configure in controller section) - redisLocation: "on-cluster" - # Set the location of Workflow's rabbitmq instance + # Set the location of Workflow's valkey instance # Valid values are: - # - on-cluster: Run Rabbitmq within the Kubernetes cluster - # - off-cluster: Run Rabbitmq outside the Kubernetes cluster (configure in controller section) - rabbitmqLocation: "on-cluster" + # - on-cluster: Run Valkey within the Kubernetes cluster + # - off-cluster: Run Valkey outside the Kubernetes cluster (configure in controller section) + valkeyLocation: "on-cluster" # Enable usage of RBAC authorization mode # # Valid values are: diff --git a/rootfs/helmbroker/celery.py b/rootfs/helmbroker/celery.py index 67f810f..45d4181 100644 --- a/rootfs/helmbroker/celery.py +++ b/rootfs/helmbroker/celery.py @@ -1,28 +1,32 @@ import os +from urllib.parse import urlparse, parse_qs, urlencode from kombu import Exchange, Queue from celery import Celery +from .config import VALKEY_URL class Config(object): # Celery Configuration Options - timezone = "Asia/Shanghai" enable_utc = True task_serializer = 'pickle' accept_content = frozenset([ - 'application/data', - 'application/text', - 'application/json', - 'application/x-python-serialize', + 'application/data', + 'application/text', + 'application/json', + 'application/x-python-serialize', ]) task_track_started = True task_time_limit = 30 * 60 worker_max_tasks_per_child = 200 + worker_prefetch_multiplier = 1 result_expires = 24 * 60 * 60 - broker_url = os.environ.get("HELMBROKER_RABBITMQ_URL", 'amqp://guest:guest@127.0.0.1:5672/') - broker_connection_retry_on_startup = True - task_default_queue = 'helmbroker.low' + cache_backend = 'django-cache' + task_default_queue = 'helmbroker.middle' task_default_exchange = 'helmbroker.priority' - task_default_routing_key = 'helmbroker.priority.low' + task_default_routing_key = 'helmbroker.priority.middle' + broker_transport_options = {"queue_order_strategy": "sorted"} + task_create_missing_queues = True + task_inherit_parent_priority = True broker_connection_retry_on_startup = True worker_cancel_long_running_tasks_on_connection_loss = True @@ -30,6 +34,7 @@ class Config(object): app = Celery('helmbroker') app.config_from_object(Config()) app.conf.update( + timezone=os.environ.get('TZ', 'UTC'), task_routes={ 'helmbroker.tasks.provision': { 'queue': 'helmbroker.high', @@ -51,22 +56,36 @@ class Config(object): task_queues=( Queue( 'helmbroker.low', exchange=Exchange('helmbroker.priority', type="direct"), - routing_key='helmbroker.priority.low', queue_arguments={'x-queue-type': 'quorum'}, + routing_key='helmbroker.priority.low', ), Queue( 'helmbroker.high', exchange=Exchange('helmbroker.priority', type="direct"), - routing_key='helmbroker.priority.high', queue_arguments={'x-queue-type': 'quorum'}, + routing_key='helmbroker.priority.high', ), Queue( 'helmbroker.middle', exchange=Exchange('helmbroker.priority', type="direct"), - routing_key='helmbroker.priority.middle', queue_arguments={'x-queue-type': 'quorum'}, + routing_key='helmbroker.priority.middle', ), ), ) app.autodiscover_tasks(("helmbroker.tasks",)) - - -app.config_from_object(Config()) +url = urlparse(VALKEY_URL) +query = parse_qs(url.query) +broker_transport_options = {"queue_order_strategy": "sorted", "visibility_timeout": 43200} +result_backend_transport_options = {} +if 'master_set' in query: + master_name = query.pop('master_set')[0] + password = url.netloc.split("@")[0].split(":")[1] + kwargs = {'sentinel_kwargs': {'password': password}, 'master_name': master_name} + broker_transport_options.update(kwargs) + result_backend_transport_options.update(kwargs) + VALKEY_URL = f"sentinel://{url.netloc}{url.path}?{urlencode(query)}" +app.conf.update( + broker_url=VALKEY_URL, + result_backend=VALKEY_URL, + broker_transport_options=broker_transport_options, + result_backend_transport_options=result_backend_transport_options, +) if __name__ == '__main__': app.start() diff --git a/rootfs/helmbroker/config.py b/rootfs/helmbroker/config.py index 6def522..a3e9f47 100644 --- a/rootfs/helmbroker/config.py +++ b/rootfs/helmbroker/config.py @@ -10,7 +10,7 @@ USERNAME = os.environ.get('HELMBROKER_USERNAME') PASSWORD = os.environ.get('HELMBROKER_PASSWORD') -REDIS_URL = os.environ.get("HELMBROKER_REDIS_URL", 'redis://localhost:6379/0') +VALKEY_URL = os.environ.get("HELMBROKER_VALKEY_URL", 'redis://localhost:6379/0') class Config: diff --git a/rootfs/helmbroker/database/fetch.py b/rootfs/helmbroker/database/fetch.py index 4fe252f..d2d8102 100644 --- a/rootfs/helmbroker/database/fetch.py +++ b/rootfs/helmbroker/database/fetch.py @@ -58,7 +58,7 @@ def _fetch_addon(url, dest): os.makedirs(dest, exist_ok=True) with tarfile.open(fileobj=tgz_file, mode="r:gz") as tarobj: for tarinfo in tarobj: - tarobj.extract(tarinfo.name, dest) + tarobj.extract(tarinfo.name, dest, filter='data') filename1 = os.path.join(dest, "meta.yaml") with open(filename1, "r") as f1: meta = yaml.load(stream=f1, Loader=yaml.Loader) diff --git a/rootfs/helmbroker/database/metadata.py b/rootfs/helmbroker/database/metadata.py index 6b37041..2877aaa 100644 --- a/rootfs/helmbroker/database/metadata.py +++ b/rootfs/helmbroker/database/metadata.py @@ -4,8 +4,8 @@ import logging import jsonschema -from redis import client -from ..config import ADDONS_PATH, REDIS_URL +from ..utils import get_valkey_client +from ..config import ADDONS_PATH logger = logging.getLogger(__name__) @@ -116,21 +116,20 @@ def save_instance_meta(instance_id, data): json_data = json.dumps(data, sort_keys=True, indent=2) with open(file, "w") as f: f.write(json_data) - redis = client.Redis.from_url(REDIS_URL) - redis.set(cache_key, json_data) + get_valkey_client().set(cache_key, json_data) def load_instance_meta(instance_id): cache_key = f"helmbroker:instance:{instance_id}" - redis = client.Redis.from_url(REDIS_URL) + valkey = get_valkey_client() - json_data = redis.get(cache_key) + json_data = valkey.get(cache_key) if not json_data: from .query import get_instance_file file = get_instance_file(instance_id) with open(file) as f: json_data = f.read() - redis.set(cache_key, json_data) + valkey.set(cache_key, json_data) return json.loads(json_data) @@ -144,20 +143,19 @@ def save_binding_meta(instance_id, data): json_data = json.dumps(data, sort_keys=True, indent=2) with open(file, "w") as f: f.write(json_data) - redis = client.Redis.from_url(REDIS_URL) - redis.set(cache_key, json_data) + get_valkey_client().set(cache_key, json_data) def load_binding_meta(instance_id): from .query import get_binding_file cache_key = f"helmbroker:binding:{instance_id}" - redis = client.Redis.from_url(REDIS_URL) - json_data = redis.get(cache_key) + valkey = get_valkey_client() + json_data = valkey.get(cache_key) if not json_data: file = get_binding_file(instance_id) with open(file, 'r') as f: json_data = f.read() - redis.set(cache_key, json_data) + valkey.set(cache_key, json_data) return json.loads(json_data) @@ -170,18 +168,17 @@ def save_addons_meta(data): json_data = json.dumps(data, sort_keys=True, indent=2) with open(file, "w") as f: f.write(json_data) - redis = client.Redis.from_url(REDIS_URL) - redis.set(cache_key, json_data) + get_valkey_client().set(cache_key, json_data) def load_addons_meta(): cache_key = "helmbroker:addons" - redis = client.Redis.from_url(REDIS_URL) + valkey = get_valkey_client() - json_data = redis.get(cache_key) + json_data = valkey.get(cache_key) if not json_data: file = os.path.join(ADDONS_PATH, "addons.json") with open(file, 'r') as f: json_data = f.read() - redis.set(cache_key, json_data) + valkey.set(cache_key, json_data) return json.loads(json_data) diff --git a/rootfs/helmbroker/utils.py b/rootfs/helmbroker/utils.py index ff19967..b76e1b4 100644 --- a/rootfs/helmbroker/utils.py +++ b/rootfs/helmbroker/utils.py @@ -5,13 +5,13 @@ import base64 import copy import logging - +from urllib.parse import urlparse, parse_qs from contextlib import contextmanager -from redis import client -from .config import REDIS_URL +from redis.client import Redis +from redis.sentinel import Sentinel +from .config import VALKEY_URL logger = logging.getLogger(__name__) - REGISTRY_CONFIG_SUFFIX = '.config/helm/registry.json' REPOSITORY_CACHE_SUFFIX = '.cache/helm/repository' REPOSITORY_CONFIG_SUFFIX = '.config/helm/repository' @@ -42,9 +42,23 @@ def helm(instance_id, *args, output_type="text"): return command("helm", *new_args, output_type=output_type) +def get_valkey_client(): + url = urlparse(VALKEY_URL) + query = parse_qs(url.query) + if 'master_set' in query: + user, host = url.netloc.split("@") + password = user.split(":")[1] + sentinel = Sentinel( + [host.split(":")], + sentinel_kwargs={'password': password}, + password=password, + ) + return sentinel.master_for(query['master_set'][0], socket_timeout=1) + return Redis.from_url(VALKEY_URL) + + def new_instance_lock(instance_id): - redis = client.Redis.from_url(REDIS_URL) - return redis.lock(instance_id) + return get_valkey_client().lock(instance_id) @contextmanager diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt index 1141fae..df9e44a 100644 --- a/rootfs/requirements.txt +++ b/rootfs/requirements.txt @@ -2,6 +2,6 @@ PyYAML==6.0.2 gunicorn==23.0.0 openbrokerapi==4.7.1 requests==2.32.2 -celery==5.5.0rc1 +celery==5.4.0 redis==5.2.0 jsonschema==4.23.0 From 692aab2bcc76161dd9ed326d1a99ef5f2cb53009 Mon Sep 17 00:00:00 2001 From: lijianguo Date: Wed, 27 Nov 2024 13:43:38 +0800 Subject: [PATCH 13/28] chore(helmbroker): update state before upgrade --- rootfs/helmbroker/broker.py | 13 ++++++++----- rootfs/helmbroker/tasks.py | 9 ++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/rootfs/helmbroker/broker.py b/rootfs/helmbroker/broker.py index 8b38269..4ba8eb8 100644 --- a/rootfs/helmbroker/broker.py +++ b/rootfs/helmbroker/broker.py @@ -1,6 +1,6 @@ import os -import shutil import logging +import time from typing import Union, List, Optional from openbrokerapi.catalog import ServicePlan @@ -18,7 +18,8 @@ from .database.query import get_instance_path, get_chart_path, get_plan_path, \ get_addon_updateable, get_addon_bindable, get_addon_allow_params, \ get_addon_archive, get_binding_file, get_instance_file -from .database.metadata import load_instance_meta, load_binding_meta, load_addons_meta +from .database.metadata import load_instance_meta, load_binding_meta, load_addons_meta, \ + save_instance_meta from .tasks import provision, bind, deprovision, update, unbind logger = logging.getLogger(__name__) @@ -94,9 +95,6 @@ def bind(self, instance_path = get_instance_path(instance_id) if os.path.exists(f'{instance_path}/bind.json'): raise ErrBindingAlreadyExists() - chart_path, plan_path = ( - get_chart_path(instance_id), get_plan_path(instance_id)) - shutil.copy(f'{plan_path}/bind.yaml', f'{chart_path}/templates') bind(instance_id, binding_id, details, async_allowed, **kwargs) data = load_binding_meta(instance_id) if data["last_operation"]["state"] == OperationState.SUCCEEDED.value: @@ -145,6 +143,11 @@ def update(self, if details.plan_id is not None: chart_path, plan_path = get_chart_path(instance_id), get_plan_path(instance_id) fetch_chart_plan(details.service_id, chart_path, details.plan_id, plan_path) + data = load_instance_meta(instance_id) + data['last_operation']["state"] = OperationState.IN_PROGRESS.value + data['last_operation']["description"] = ( + f"update {instance_id} in progress at {time.time()}") + save_instance_meta(instance_id, data) update.delay(instance_id, details) return UpdateServiceSpec(is_async=True) diff --git a/rootfs/helmbroker/tasks.py b/rootfs/helmbroker/tasks.py index 8f19af7..750e603 100644 --- a/rootfs/helmbroker/tasks.py +++ b/rootfs/helmbroker/tasks.py @@ -2,6 +2,7 @@ import time import yaml import logging +import shutil from openbrokerapi.service_broker import ProvisionDetails, OperationState, \ UpdateDetails, BindDetails @@ -99,10 +100,6 @@ def update(instance_id: str, details: UpdateDetails): data["last_operation"]["description"] = f"update {instance_id} failed: {output}" save_instance_meta(instance_id, data) return - data['last_operation']["state"] = OperationState.IN_PROGRESS.value - data['last_operation']["description"] = ( - f"update {instance_id} in progress at {time.time()}") - save_instance_meta(instance_id, data) chart_path = get_chart_path(instance_id) values_file = os.path.join(get_plan_path(instance_id), "values.yaml") args = [ @@ -148,7 +145,9 @@ def bind(instance_id: str, save_binding_meta(instance_id, data) return save_binding_meta(instance_id, data) - chart_path = get_chart_path(instance_id) + chart_path, plan_path = ( + get_chart_path(instance_id), get_plan_path(instance_id)) + shutil.copy(f'{plan_path}/bind.yaml', f'{chart_path}/templates') values_file = os.path.join(get_plan_path(instance_id), "values.yaml") args = [ "template", details.context["instance_name"], chart_path, From 9f223bd98419250d2fa01f477f429d55ddb88a91 Mon Sep 17 00:00:00 2001 From: lijianguo Date: Fri, 6 Dec 2024 16:14:57 +0800 Subject: [PATCH 14/28] chore(helmbroker): add debug config --- rootfs/helmbroker/broker.py | 8 +++++++- rootfs/helmbroker/config.py | 2 +- rootfs/helmbroker/gunicorn/logging.py | 4 ++-- rootfs/helmbroker/tasks.py | 12 +++++++++++- rootfs/helmbroker/wsgi.py | 3 ++- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/rootfs/helmbroker/broker.py b/rootfs/helmbroker/broker.py index 4ba8eb8..c9bfa9a 100644 --- a/rootfs/helmbroker/broker.py +++ b/rootfs/helmbroker/broker.py @@ -43,6 +43,7 @@ def provision(self, details: ProvisionDetails, async_allowed: bool, **kwargs) -> ProvisionedServiceSpec: + logger.debug(f"*** provision instance {instance_id}") instance_path = get_instance_path(instance_id) if os.path.exists(instance_path): raise ErrInstanceAlreadyExists() @@ -83,6 +84,7 @@ def bind(self, async_allowed: bool, **kwargs ) -> Binding: + logger.debug(f"*** bind instance {instance_id}") is_addon_bindable = get_addon_bindable(details.service_id) if not is_addon_bindable: raise ErrBadRequest( @@ -110,7 +112,7 @@ def unbind(self, async_allowed: bool, **kwargs ) -> UnbindSpec: - logger.debug(f"unbind instance {instance_id}") + logger.debug(f"*** unbind instance {instance_id}") unbind.delay(instance_id) return UnbindSpec(is_async=True) @@ -120,6 +122,7 @@ def update(self, async_allowed: bool, **kwargs ) -> UpdateServiceSpec: + logger.debug(f"*** update instance {instance_id}") instance_path = get_instance_path(instance_id) if not os.path.exists(instance_path): raise ErrBadRequest(msg="Instance %s does not exist" % instance_id) @@ -156,6 +159,7 @@ def deprovision(self, details: DeprovisionDetails, async_allowed: bool, **kwargs) -> DeprovisionServiceSpec: + logger.debug(f"*** deprovision instance {instance_id}") if not os.path.exists(get_instance_path(instance_id)): raise ErrInstanceDoesNotExist() with new_instance_lock(instance_id): @@ -175,6 +179,7 @@ def last_operation(self, operation_data: Optional[str], **kwargs ) -> LastOperation: + logger.debug(f"*** last_operation instance {instance_id}") if os.path.exists(get_instance_file(instance_id)): data = load_instance_meta(instance_id) return LastOperation( @@ -189,6 +194,7 @@ def last_binding_operation(self, operation_data: Optional[str], **kwargs ) -> LastOperation: + logger.debug(f"*** last_binding_operation instance {instance_id}") if os.path.exists(get_binding_file(instance_id)): data = load_binding_meta(instance_id) return LastOperation( diff --git a/rootfs/helmbroker/config.py b/rootfs/helmbroker/config.py index a3e9f47..e5becfa 100644 --- a/rootfs/helmbroker/config.py +++ b/rootfs/helmbroker/config.py @@ -14,4 +14,4 @@ class Config: - DEBUG = bool(os.environ.get('HELMBROKER_DEBUG', True)) + DEBUG = bool(os.environ.get('HELMBROKER_DEBUG', False)) diff --git a/rootfs/helmbroker/gunicorn/logging.py b/rootfs/helmbroker/gunicorn/logging.py index f9e65f3..41a3f9e 100644 --- a/rootfs/helmbroker/gunicorn/logging.py +++ b/rootfs/helmbroker/gunicorn/logging.py @@ -1,12 +1,12 @@ -import os from gunicorn.glogging import Logger +from helmbroker.config import Config class Logging(Logger): def access(self, resp, req, environ, request_time): # health check endpoints are only logged in debug mode if ( - not os.environ.get('DEBUG', False) and + not Config.DEBUG and req.path in ['/readiness', '/healthz'] ): return diff --git a/rootfs/helmbroker/tasks.py b/rootfs/helmbroker/tasks.py index 750e603..69f2f30 100644 --- a/rootfs/helmbroker/tasks.py +++ b/rootfs/helmbroker/tasks.py @@ -20,10 +20,12 @@ @app.task(serializer='pickle') def provision(instance_id: str, details: ProvisionDetails): + logger.debug(f"*** task provision instance: {instance_id}, before lock") with ( new_instance_lock(instance_id), run_instance_hooks(instance_id, "provision") as (status, output) ): + logger.debug(f"*** task provision instance: {instance_id}") backup_instance(instance_id) # create instance.json data = { @@ -78,10 +80,12 @@ def provision(instance_id: str, details: ProvisionDetails): @app.task(serializer='pickle') def update(instance_id: str, details: UpdateDetails): + logger.debug(f"*** task update instance: {instance_id}, before lock") with ( new_instance_lock(instance_id), run_instance_hooks(instance_id, "update") as (status, output) ): + logger.debug(f"*** task update instance: {instance_id}") backup_instance(instance_id) data = load_instance_meta(instance_id) if details.service_id: @@ -133,10 +137,12 @@ def bind(instance_id: str, details: BindDetails, async_allowed: bool, **kwargs): + logger.debug(f"*** task bind instance: {instance_id}, before lock") with ( new_instance_lock(instance_id), run_instance_hooks(instance_id, "bind") as (status, output) ): + logger.debug(f"*** task bind instance: {instance_id}") backup_instance(instance_id) data = {"binding_id": binding_id, "credentials": {}, "last_operation": {}} if status != 0: @@ -199,10 +205,12 @@ def bind(instance_id: str, @app.task(serializer='pickle') def unbind(instance_id): + logger.debug(f"*** task unbind instance: {instance_id}, before lock") with ( new_instance_lock(instance_id), - run_instance_hooks(instance_id, "deprovision") as (status, output) + run_instance_hooks(instance_id, "unbind") as (status, output) ): + logger.debug(f"*** task unbind instance: {instance_id}") backup_instance(instance_id) data = load_binding_meta(instance_id) if status != 0: @@ -220,10 +228,12 @@ def unbind(instance_id): @app.task(serializer='pickle') def deprovision(instance_id: str): + logger.debug(f"*** task deprovision instance: {instance_id}, before lock") with ( new_instance_lock(instance_id), run_instance_hooks(instance_id, "deprovision") as (status, output) ): + logger.debug(f"*** task deprovision instance: {instance_id}") backup_instance(instance_id) data = load_instance_meta(instance_id) if status != 0: diff --git a/rootfs/helmbroker/wsgi.py b/rootfs/helmbroker/wsgi.py index ac89884..64db221 100644 --- a/rootfs/helmbroker/wsgi.py +++ b/rootfs/helmbroker/wsgi.py @@ -1,4 +1,5 @@ import os +import logging from flask import Flask, make_response from openbrokerapi import api, log_util from helmbroker.broker import HelmServiceBroker @@ -27,5 +28,5 @@ def readiness(): catalog_api = api.get_blueprint( HelmServiceBroker(), api.BrokerCredentials(USERNAME, PASSWORD), - log_util.basic_config()) + log_util.basic_config(level=logging.DEBUG if Config.DEBUG else logging.INFO)) application.register_blueprint(catalog_api) From 4d0b65494ec512d096c5256ca806757fbb6dae1a Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Mon, 23 Dec 2024 09:19:11 +0800 Subject: [PATCH 15/28] fix(wooddpecker): secsets are deprecated --- .woodpecker/build-linux.yml | 17 +++++++++++------ .woodpecker/chart.yaml | 14 +++++++++----- .woodpecker/manifest.yml | 13 ++++++++----- .woodpecker/test-linux.yml | 11 +++++++---- 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/.woodpecker/build-linux.yml b/.woodpecker/build-linux.yml index 782d2f5..3424a97 100644 --- a/.woodpecker/build-linux.yml +++ b/.woodpecker/build-linux.yml @@ -14,12 +14,17 @@ steps: - export VERSION=$([ -z $CI_COMMIT_TAG ] && echo latest || echo $CI_COMMIT_TAG)-$(sed 's#/#-#g' <<< $CI_SYSTEM_PLATFORM) - echo $CONTAINER_PASSWORD | podman login $DRYCC_REGISTRY --username $CONTAINER_USERNAME --password-stdin > /dev/null 2>&1 - make podman-build podman-immutable-push - secrets: - - codename - - dev_registry - - drycc_registry - - container_username - - container_password + environment: + CODENAME: + from_secret: codename + DEV_REGISTRY: + from_secret: dev_registry + DRYCC_REGISTRY: + from_secret: drycc_registry + CONTAINER_USERNAME: + from_secret: container_username + CONTAINER_PASSWORD: + from_secret: container_password when: event: - push diff --git a/.woodpecker/chart.yaml b/.woodpecker/chart.yaml index 67df5de..6ee4b57 100644 --- a/.woodpecker/chart.yaml +++ b/.woodpecker/chart.yaml @@ -15,11 +15,15 @@ steps: - helm package -u charts/$${CI_REPO_NAME} --version $CHART_VERSION --app-version $APP_VERSION - echo $CONTAINER_PASSWORD | helm registry login $DRYCC_REGISTRY -u $CONTAINER_USERNAME --password-stdin - helm push $${CI_REPO_NAME}-$CHART_VERSION.tgz oci://$DRYCC_REGISTRY/$([ -z $CI_COMMIT_TAG ] && echo charts-testing || echo charts) - secrets: - - dev_registry - - drycc_registry - - container_username - - container_password + environment: + DEV_REGISTRY: + from_secret: dev_registry + DRYCC_REGISTRY: + from_secret: drycc_registry + CONTAINER_USERNAME: + from_secret: container_username + CONTAINER_PASSWORD: + from_secret: container_password when: event: - push diff --git a/.woodpecker/manifest.yml b/.woodpecker/manifest.yml index b4735ee..010bcdd 100644 --- a/.woodpecker/manifest.yml +++ b/.woodpecker/manifest.yml @@ -8,8 +8,9 @@ steps: commands: - sed -i "s/{{project}}/$${CI_REPO_NAME}/g" .woodpecker/manifest.tmpl - sed -i "s/registry.drycc.cc/$${DRYCC_REGISTRY}/g" .woodpecker/manifest.tmpl - secrets: - - drycc_registry + environment: + DRYCC_REGISTRY: + from_secret: drycc_registry when: event: - tag @@ -26,9 +27,11 @@ steps: -v $(pwd):$(pwd) -w $(pwd) docker.io/plugins/manifest - secrets: - - container_username - - container_password + environment: + CONTAINER_USERNAME: + from_secret: container_username + CONTAINER_PASSWORD: + from_secret: container_password when: event: - tag diff --git a/.woodpecker/test-linux.yml b/.woodpecker/test-linux.yml index 16a01b5..9a06860 100644 --- a/.woodpecker/test-linux.yml +++ b/.woodpecker/test-linux.yml @@ -12,10 +12,13 @@ steps: image: bash commands: - make test upload-coverage - secrets: - - codename - - dev_registry - - codecov_token + environment: + CODENAME: + from_secret: codename + DEV_REGISTRY: + from_secret: dev_registry + CODECOV_TOKEN: + from_secret: codecov_token when: event: - push From c11d9a6fb67c25f8c7c50da084a977bf6ed04e50 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 8 Apr 2025 16:40:26 +0800 Subject: [PATCH 16/28] chore(helmbroker): bump latest version --- rootfs/Dockerfile | 6 +++--- rootfs/Dockerfile.test | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index e97938a..0c84b11 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -4,9 +4,9 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/workspace \ - PYTHON_VERSION="3.12" \ - HELM_VERSION="3.15.3" \ - KUBECTL_VERSION="1.30.3" + PYTHON_VERSION="3.13" \ + HELM_VERSION="3.17.2" \ + KUBECTL_VERSION="1.32.3" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test index b5c1d34..8c8d2a3 100644 --- a/rootfs/Dockerfile.test +++ b/rootfs/Dockerfile.test @@ -4,9 +4,9 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/workspace \ - PYTHON_VERSION="3.12" \ - HELM_VERSION="3.15.3" \ - KUBECTL_VERSION="1.30.3" + PYTHON_VERSION="3.13" \ + HELM_VERSION="3.17.2" \ + KUBECTL_VERSION="1.32.3" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} From ab96f3c0295acbc074711eaa1d01d8925fe60e49 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 8 Apr 2025 16:54:18 +0800 Subject: [PATCH 17/28] chore(woodpecker): add cron event --- .woodpecker/build-linux.yml | 1 + .woodpecker/chart.yaml | 1 + .woodpecker/manifest.yml | 2 ++ .woodpecker/test-linux.yml | 1 + rootfs/Dockerfile | 2 +- rootfs/Dockerfile.test | 2 +- 6 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.woodpecker/build-linux.yml b/.woodpecker/build-linux.yml index 3424a97..ef4ecfd 100644 --- a/.woodpecker/build-linux.yml +++ b/.woodpecker/build-linux.yml @@ -29,6 +29,7 @@ steps: event: - push - tag + - cron depends_on: - test-linux \ No newline at end of file diff --git a/.woodpecker/chart.yaml b/.woodpecker/chart.yaml index 6ee4b57..d40739c 100644 --- a/.woodpecker/chart.yaml +++ b/.woodpecker/chart.yaml @@ -28,6 +28,7 @@ steps: event: - push - tag + - cron depends_on: - manifest \ No newline at end of file diff --git a/.woodpecker/manifest.yml b/.woodpecker/manifest.yml index 010bcdd..7330ea0 100644 --- a/.woodpecker/manifest.yml +++ b/.woodpecker/manifest.yml @@ -15,6 +15,7 @@ steps: event: - tag - push + - cron - name: publish-manifest image: bash @@ -36,6 +37,7 @@ steps: event: - tag - push + - cron depends_on: - build-linux diff --git a/.woodpecker/test-linux.yml b/.woodpecker/test-linux.yml index 9a06860..ab8ee69 100644 --- a/.woodpecker/test-linux.yml +++ b/.woodpecker/test-linux.yml @@ -23,3 +23,4 @@ steps: event: - push - tag + - cron diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 0c84b11..b6dad30 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -4,7 +4,7 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/workspace \ - PYTHON_VERSION="3.13" \ + PYTHON_VERSION="3.12" \ HELM_VERSION="3.17.2" \ KUBECTL_VERSION="1.32.3" diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test index 8c8d2a3..7596355 100644 --- a/rootfs/Dockerfile.test +++ b/rootfs/Dockerfile.test @@ -4,7 +4,7 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/workspace \ - PYTHON_VERSION="3.13" \ + PYTHON_VERSION="3.12" \ HELM_VERSION="3.17.2" \ KUBECTL_VERSION="1.32.3" From e7375a7e8e9c177c6888fd87d4d6a065a0ef1c80 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Mon, 12 May 2025 12:37:06 +0800 Subject: [PATCH 18/28] chore(charts): change resources format --- charts/helmbroker/templates/_helpers.tpl | 14 -------------- .../templates/helmbroker-celery-deployment.yaml | 5 ++++- .../templates/helmbroker-deployment.yaml | 5 ++++- charts/helmbroker/values.yaml | 16 ++++++++++++++-- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/charts/helmbroker/templates/_helpers.tpl b/charts/helmbroker/templates/_helpers.tpl index 4702bc6..8cb0ee7 100644 --- a/charts/helmbroker/templates/_helpers.tpl +++ b/charts/helmbroker/templates/_helpers.tpl @@ -28,20 +28,6 @@ env: {{- end }} {{- end }} -{{/* Generate helmbroker deployment limits */}} -{{- define "helmbroker.limits" -}} -{{- if or (.Values.limitsCpu) (.Values.limitsMemory) }} -resources: - limits: -{{- if (.Values.limitsCpu) }} - cpu: {{.Values.limitsCpu}} -{{- end }} -{{- if (.Values.limitsMemory) }} - memory: {{.Values.limitsMemory}} -{{- end }} -{{- end }} -{{- end }} - {{/* Generate helmbroker deployment volumeMounts */}} {{- define "helmbroker.volumeMounts" }} volumeMounts: diff --git a/charts/helmbroker/templates/helmbroker-celery-deployment.yaml b/charts/helmbroker/templates/helmbroker-celery-deployment.yaml index 7388b2c..d2b4ab5 100644 --- a/charts/helmbroker/templates/helmbroker-celery-deployment.yaml +++ b/charts/helmbroker/templates/helmbroker-celery-deployment.yaml @@ -49,7 +49,10 @@ spec: - -c - celery --app helmbroker worker --queues helmbroker.low,helmbroker.middle,helmbroker.high --autoscale=32,1 --loglevel=WARNING {{- end }} - {{- include "helmbroker.limits" $ | indent 8 }} + {{- with index .Values "celery" "resources" }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} {{- include "helmbroker.envs" $ | indent 8 }} {{- include "helmbroker.volumeMounts" $ | indent 8 }} {{- include "helmbroker.volumes" . | indent 6 }} diff --git a/charts/helmbroker/templates/helmbroker-deployment.yaml b/charts/helmbroker/templates/helmbroker-deployment.yaml index 6ad1385..6659e7d 100644 --- a/charts/helmbroker/templates/helmbroker-deployment.yaml +++ b/charts/helmbroker/templates/helmbroker-deployment.yaml @@ -71,7 +71,10 @@ spec: ports: - containerPort: 8000 name: http - {{- include "helmbroker.limits" . | indent 8 }} + {{- with index .Values "api" "resources" }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} {{- include "helmbroker.envs" . | indent 8 }} {{- include "helmbroker.volumeMounts" . | indent 8 }} {{- include "helmbroker.volumes" . | indent 6 }} diff --git a/charts/helmbroker/values.yaml b/charts/helmbroker/values.yaml index 60915f7..3af3f08 100644 --- a/charts/helmbroker/values.yaml +++ b/charts/helmbroker/values.yaml @@ -3,8 +3,6 @@ imageTag: "canary" imageRegistry: "registry.drycc.cc" imagePullPolicy: "Always" replicas: 1 -# limitsCpu: "100m" -# limitsMemory: "50Mi" ## Enable diagnostic mode ## @@ -44,6 +42,13 @@ environment: # HELMBROKER_CONFIG_ROOT: /etc/helmbroker api: + resources: {} + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi nodeAffinityPreset: key: "drycc.cc/node" type: "soft" @@ -59,6 +64,13 @@ api: app: "drycc-helmbroker" celery: + resources: {} + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi nodeAffinityPreset: key: "drycc.cc/node" type: "soft" From 5f8fd3c19705f1734f0c0b0c9a5849dca1008f70 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 15 Jul 2025 23:17:54 +0800 Subject: [PATCH 19/28] chore(workflow): remove cluster domain --- charts/helmbroker/Chart.yaml | 1 + charts/helmbroker/templates/_helpers.tpl | 4 ++-- .../helmbroker-celery-deployment.yaml | 4 ++-- charts/helmbroker/values.yaml | 19 ++----------------- rootfs/helmbroker/wsgi.py | 3 +-- 5 files changed, 8 insertions(+), 23 deletions(-) diff --git a/charts/helmbroker/Chart.yaml b/charts/helmbroker/Chart.yaml index 6e22e3a..6e8d2aa 100644 --- a/charts/helmbroker/Chart.yaml +++ b/charts/helmbroker/Chart.yaml @@ -9,6 +9,7 @@ dependencies: - name: valkey repository: {{repository}} version: x.x.x + condition: valkey.enabled description: Drycc Workflow helmbroker. maintainers: - name: Drycc Team diff --git a/charts/helmbroker/templates/_helpers.tpl b/charts/helmbroker/templates/_helpers.tpl index 8cb0ee7..3e9e40f 100644 --- a/charts/helmbroker/templates/_helpers.tpl +++ b/charts/helmbroker/templates/_helpers.tpl @@ -13,14 +13,14 @@ env: secretKeyRef: name: helmbroker-creds key: valkey-url -{{- else if eq .Values.global.valkeyLocation "on-cluster" }} +{{- else if .Values.valkey.enabled }} - name: VALKEY_PASSWORD valueFrom: secretKeyRef: name: valkey-creds key: password - name: HELMBROKER_VALKEY_URL - value: "redis://:$(VALKEY_PASSWORD)@drycc-valkey.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:26379/0?master_set=drycc" + value: "redis://:$(VALKEY_PASSWORD)@drycc-valkey:26379/0?master_set=drycc" {{- end }} {{- range $key, $value := .Values.environment }} - name: {{ $key }} diff --git a/charts/helmbroker/templates/helmbroker-celery-deployment.yaml b/charts/helmbroker/templates/helmbroker-celery-deployment.yaml index d2b4ab5..d188ba5 100644 --- a/charts/helmbroker/templates/helmbroker-celery-deployment.yaml +++ b/charts/helmbroker/templates/helmbroker-celery-deployment.yaml @@ -33,8 +33,8 @@ spec: args: - netcat - -v - - -a - - $(DRYCC_HELMBROKER_SERVICE_HOST):$(DRYCC_HELMBROKER_SERVICE_PORT) + - -u + - http://drycc-helmbroker {{- include "helmbroker.envs" . | indent 10 }} containers: - name: drycc-helmbroker-celery diff --git a/charts/helmbroker/values.yaml b/charts/helmbroker/values.yaml index 3af3f08..31fee4e 100644 --- a/charts/helmbroker/values.yaml +++ b/charts/helmbroker/values.yaml @@ -95,20 +95,5 @@ persistence: storageClass: "" volumeName: "" -global: - # Set the location of Workflow's valkey instance - # Valid values are: - # - on-cluster: Run Valkey within the Kubernetes cluster - # - off-cluster: Run Valkey outside the Kubernetes cluster (configure in controller section) - valkeyLocation: "on-cluster" - # Enable usage of RBAC authorization mode - # - # Valid values are: - # - true: all RBAC-related manifests will be installed (in case your cluster supports RBAC) - # - false: no RBAC-related manifests will be installed - rbac: true - # A domain name consists of one or more parts. - # Periods (.) are used to separate these parts. - # Each part must be 1 to 63 characters in length and can contain lowercase letters, digits, and hyphens (-). - # It must start and end with a lowercase letter or digit. - clusterDomain: "cluster.local" +valkey: + enabled: true diff --git a/rootfs/helmbroker/wsgi.py b/rootfs/helmbroker/wsgi.py index 64db221..206c711 100644 --- a/rootfs/helmbroker/wsgi.py +++ b/rootfs/helmbroker/wsgi.py @@ -18,8 +18,7 @@ def readiness(): if "KUBECONFIG" in os.environ: return "OK" elif "KUBERNETES_SERVICE_PORT" in os.environ and \ - ("KUBERNETES_SERVICE_HOST" in os.environ or - "KUBERNETES_CLUSTER_DOMAIN" in os.environ): + "KUBERNETES_SERVICE_HOST" in os.environ: return "OK" return make_response("kubernetes not available", 500) From bf76ea5c889ae36c5cd065e391b240f9a167753b Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 2 Sep 2025 09:39:02 +0800 Subject: [PATCH 20/28] chore(charts): celery process isolation --- .../templates/helmbroker-celery-deployment.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/charts/helmbroker/templates/helmbroker-celery-deployment.yaml b/charts/helmbroker/templates/helmbroker-celery-deployment.yaml index d188ba5..e3a3d67 100644 --- a/charts/helmbroker/templates/helmbroker-celery-deployment.yaml +++ b/charts/helmbroker/templates/helmbroker-celery-deployment.yaml @@ -37,7 +37,8 @@ spec: - http://drycc-helmbroker {{- include "helmbroker.envs" . | indent 10 }} containers: - - name: drycc-helmbroker-celery + {{- range $key := (list "low" "middle" "high") }} + - name: drycc-helmbroker-celery-{{$key}} image: {{$.Values.imageRegistry}}/{{$.Values.imageOrg}}/helmbroker:{{$.Values.imageTag}} imagePullPolicy: {{$.Values.imagePullPolicy}} {{- if $.Values.diagnosticMode.enabled }} @@ -47,12 +48,13 @@ spec: args: - /bin/bash - -c - - celery --app helmbroker worker --queues helmbroker.low,helmbroker.middle,helmbroker.high --autoscale=32,1 --loglevel=WARNING + - celery --app helmbroker worker -n {{uuidv4}}@%h --queues helmbroker.{{$key}} --autoscale=32,1 --loglevel=WARNING {{- end }} - {{- with index .Values "celery" "resources" }} + {{- with index $.Values "celery" "resources" }} resources: {{- toYaml . | nindent 10 }} {{- end }} {{- include "helmbroker.envs" $ | indent 8 }} {{- include "helmbroker.volumeMounts" $ | indent 8 }} + {{- end }} {{- include "helmbroker.volumes" . | indent 6 }} From 1bc5efb22043f6bb9d5276df778e65eda99384b6 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 2 Sep 2025 17:11:27 +0800 Subject: [PATCH 21/28] chore(charts): change replicas location --- .../helmbroker/templates/helmbroker-celery-deployment.yaml | 2 +- charts/helmbroker/templates/helmbroker-deployment.yaml | 2 +- charts/helmbroker/values.yaml | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/helmbroker/templates/helmbroker-celery-deployment.yaml b/charts/helmbroker/templates/helmbroker-celery-deployment.yaml index e3a3d67..b584e72 100644 --- a/charts/helmbroker/templates/helmbroker-celery-deployment.yaml +++ b/charts/helmbroker/templates/helmbroker-celery-deployment.yaml @@ -7,7 +7,7 @@ metadata: annotations: component.drycc.cc/version: {{ .Values.imageTag }} spec: - replicas: {{ .Values.celeryReplicas }} + replicas: {{ .Values.celery.replicas }} strategy: rollingUpdate: maxSurge: 1 diff --git a/charts/helmbroker/templates/helmbroker-deployment.yaml b/charts/helmbroker/templates/helmbroker-deployment.yaml index 6659e7d..fe9a2ed 100644 --- a/charts/helmbroker/templates/helmbroker-deployment.yaml +++ b/charts/helmbroker/templates/helmbroker-deployment.yaml @@ -7,7 +7,7 @@ metadata: annotations: component.drycc.cc/version: {{ .Values.imageTag }} spec: - replicas: {{ .Values.replicas }} + replicas: {{ .Values.api.replicas }} strategy: rollingUpdate: maxSurge: 1 diff --git a/charts/helmbroker/values.yaml b/charts/helmbroker/values.yaml index 31fee4e..65b9f54 100644 --- a/charts/helmbroker/values.yaml +++ b/charts/helmbroker/values.yaml @@ -2,7 +2,7 @@ imageOrg: "drycc-addons" imageTag: "canary" imageRegistry: "registry.drycc.cc" imagePullPolicy: "Always" -replicas: 1 + ## Enable diagnostic mode ## @@ -24,8 +24,6 @@ repositories: - name: drycc-helm-broker url: https://github.com/drycc/addons/releases/download/latest/index.yaml -celeryReplicas: 1 - # broker_credentials: # Optional Usernames and passwords that will be required to communicate with service broker username: admin @@ -42,6 +40,7 @@ environment: # HELMBROKER_CONFIG_ROOT: /etc/helmbroker api: + replicas: 1 resources: {} # limits: # cpu: 200m @@ -64,6 +63,7 @@ api: app: "drycc-helmbroker" celery: + replicas: 1 resources: {} # limits: # cpu: 200m From c806f29bc4a96e9595e7a2ce1928ae650edd8350 Mon Sep 17 00:00:00 2001 From: lijianguo Date: Tue, 2 Sep 2025 11:03:30 +0800 Subject: [PATCH 22/28] chore(helmbroker): support validate plan schema --- rootfs/helmbroker/broker.py | 10 +++++- rootfs/helmbroker/database/query.py | 4 +++ rootfs/helmbroker/utils.py | 50 +++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/rootfs/helmbroker/broker.py b/rootfs/helmbroker/broker.py index c9bfa9a..19dda4b 100644 --- a/rootfs/helmbroker/broker.py +++ b/rootfs/helmbroker/broker.py @@ -13,7 +13,7 @@ UpdateDetails, UpdateServiceSpec, DeprovisionDetails, \ DeprovisionServiceSpec, LastOperation, OperationState -from .utils import verify_parameters, new_instance_lock +from .utils import verify_parameters, new_instance_lock, verify_parameters_by_plan from .database.fetch import fetch_chart_plan from .database.query import get_instance_path, get_chart_path, get_plan_path, \ get_addon_updateable, get_addon_bindable, get_addon_allow_params, \ @@ -64,6 +64,10 @@ def provision(self, os.makedirs(instance_path, exist_ok=True) chart_path, plan_path = get_chart_path(instance_id), get_plan_path(instance_id) fetch_chart_plan(details.service_id, chart_path, details.plan_id, plan_path) + # verify instance-schema + msg = verify_parameters_by_plan(instance_id, details.parameters) + if msg: + raise ErrBadRequest(msg) provision.delay(instance_id, details) return ProvisionedServiceSpec(state=ProvisionState.IS_ASYNC) @@ -146,6 +150,10 @@ def update(self, if details.plan_id is not None: chart_path, plan_path = get_chart_path(instance_id), get_plan_path(instance_id) fetch_chart_plan(details.service_id, chart_path, details.plan_id, plan_path) + # verify instance-schema + msg = verify_parameters_by_plan(instance_id, details.parameters) + if msg: + raise ErrBadRequest(msg) data = load_instance_meta(instance_id) data['last_operation']["state"] = OperationState.IN_PROGRESS.value data['last_operation']["description"] = ( diff --git a/rootfs/helmbroker/database/query.py b/rootfs/helmbroker/database/query.py index b7aa331..741cfc3 100644 --- a/rootfs/helmbroker/database/query.py +++ b/rootfs/helmbroker/database/query.py @@ -22,6 +22,10 @@ def get_plan_path(instance_id): return os.path.join(get_instance_path(instance_id), "plan") +def get_plan_schema_path(instance_id): + return os.path.join(get_instance_path(instance_id), "plan", "instance-schema.json") + + def get_hooks_path(instance_id): return os.path.join(get_plan_path(instance_id), "hooks") diff --git a/rootfs/helmbroker/utils.py b/rootfs/helmbroker/utils.py index b76e1b4..e749e85 100644 --- a/rootfs/helmbroker/utils.py +++ b/rootfs/helmbroker/utils.py @@ -5,6 +5,7 @@ import base64 import copy import logging +import jsonschema from urllib.parse import urlparse, parse_qs from contextlib import contextmanager from redis.client import Redis @@ -168,3 +169,52 @@ def _verify_required_parameters(allow_parameters, parameters): if error: error_parameters.add(allow_parameter["name"]) return error_parameters + + +def verify_parameters_by_plan(instance_id, parameters): + """verify parameters allowed or not""" + if not parameters: + return "" + # read schema file + from .database.query import get_plan_schema_path + schema_file = get_plan_schema_path(instance_id) + try: + with open(schema_file, 'r') as f: + schema = json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + return "" + if not schema: + return "" + # get parameters + if "rawValues" in parameters: + params = yaml.safe_load(base64.b64decode(parameters["rawValues"])) + else: + params = _convert_to_nested_dict(parameters) + # validate schema + try: + jsonschema.validate(params, schema) + except jsonschema.ValidationError as e: + return f"could not validate: {e.message}" + return "" + + +def _convert_to_nested_dict(assignments): + """ + {"a.b.c": "1Gi", "a.b.d": "2Gi"} + -> + {'a': {'b': {'c': '1Gi', 'd': '2Gi'}}} + """ + def set_nested_value(d, keys, value): + if len(keys) == 1: + d[keys[0]] = value + else: + if keys[0] not in d: + d[keys[0]] = {} + set_nested_value(d[keys[0]], keys[1:], value) + result = {} + if isinstance(assignments, dict): + # dict format: {"a.b.c": "1Gi"} + for key_path, value in assignments.items(): + keys = key_path.split('.') + set_nested_value(result, keys, value) + return result From 76b27bdb65a6907120ab93274aa4b3ae0d8dee00 Mon Sep 17 00:00:00 2001 From: jianxiaoguo Date: Fri, 12 Sep 2025 10:36:16 +0800 Subject: [PATCH 23/28] chore(helmbroker): verify instance_name length --- rootfs/helmbroker/broker.py | 5 +++++ rootfs/helmbroker/config.py | 1 + 2 files changed, 6 insertions(+) diff --git a/rootfs/helmbroker/broker.py b/rootfs/helmbroker/broker.py index 19dda4b..c920789 100644 --- a/rootfs/helmbroker/broker.py +++ b/rootfs/helmbroker/broker.py @@ -21,6 +21,7 @@ from .database.metadata import load_instance_meta, load_binding_meta, load_addons_meta, \ save_instance_meta from .tasks import provision, bind, deprovision, update, unbind +from .config import INSTANCE_NAME_LENS logger = logging.getLogger(__name__) @@ -44,6 +45,10 @@ def provision(self, async_allowed: bool, **kwargs) -> ProvisionedServiceSpec: logger.debug(f"*** provision instance {instance_id}") + # verify instance_name length + if len(details.context["instance_name"]) > INSTANCE_NAME_LENS: + raise ErrBadRequest( + msg=f"The length of the instance name cannot exceed {INSTANCE_NAME_LENS}.") instance_path = get_instance_path(instance_id) if os.path.exists(instance_path): raise ErrInstanceAlreadyExists() diff --git a/rootfs/helmbroker/config.py b/rootfs/helmbroker/config.py index e5becfa..c83ad94 100644 --- a/rootfs/helmbroker/config.py +++ b/rootfs/helmbroker/config.py @@ -11,6 +11,7 @@ PASSWORD = os.environ.get('HELMBROKER_PASSWORD') VALKEY_URL = os.environ.get("HELMBROKER_VALKEY_URL", 'redis://localhost:6379/0') +INSTANCE_NAME_LENS = int(os.environ.get("INSTANCE_NAME_LENS", '32')) class Config: From 449ca4c81bdac84469b5b7bc30844d46670722ca Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Sat, 13 Sep 2025 22:58:42 +0800 Subject: [PATCH 24/28] chore(helmbroker): bump new version --- rootfs/Dockerfile | 6 +++--- rootfs/Dockerfile.test | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index b6dad30..bd73c78 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -4,9 +4,9 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/workspace \ - PYTHON_VERSION="3.12" \ - HELM_VERSION="3.17.2" \ - KUBECTL_VERSION="1.32.3" + PYTHON_VERSION="3.13" \ + HELM_VERSION="3.19.0" \ + KUBECTL_VERSION="1.34.1" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test index 7596355..4c6eed0 100644 --- a/rootfs/Dockerfile.test +++ b/rootfs/Dockerfile.test @@ -4,9 +4,9 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/workspace \ - PYTHON_VERSION="3.12" \ - HELM_VERSION="3.17.2" \ - KUBECTL_VERSION="1.32.3" + PYTHON_VERSION="3.13" \ + HELM_VERSION="3.19.0" \ + KUBECTL_VERSION="1.34.1" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} From ebe21a9ed7b0145ce519eed0ac10b7b69e6a623b Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Thu, 18 Sep 2025 14:31:16 +0800 Subject: [PATCH 25/28] chore(helmbroker): upgrade python requirements.txt --- rootfs/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt index df9e44a..7810104 100644 --- a/rootfs/requirements.txt +++ b/rootfs/requirements.txt @@ -1,7 +1,7 @@ PyYAML==6.0.2 gunicorn==23.0.0 openbrokerapi==4.7.1 -requests==2.32.2 -celery==5.4.0 -redis==5.2.0 -jsonschema==4.23.0 +requests==2.32.5 +celery==5.5.3 +redis==6.4.0 +jsonschema==4.25.1 From 233e0584a4f2571cc212a87ed892c9d3c6d9990f Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Sat, 15 Nov 2025 20:54:47 +0800 Subject: [PATCH 26/28] chore(python): bump python version to 3.14 --- rootfs/Dockerfile | 2 +- rootfs/Dockerfile.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index bd73c78..d2661c6 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -4,7 +4,7 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/workspace \ - PYTHON_VERSION="3.13" \ + PYTHON_VERSION="3.14" \ HELM_VERSION="3.19.0" \ KUBECTL_VERSION="1.34.1" diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test index 4c6eed0..4aa8db9 100644 --- a/rootfs/Dockerfile.test +++ b/rootfs/Dockerfile.test @@ -4,7 +4,7 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/workspace \ - PYTHON_VERSION="3.13" \ + PYTHON_VERSION="3.14" \ HELM_VERSION="3.19.0" \ KUBECTL_VERSION="1.34.1" From 2d338f217f0c7051936255b35eb9cd4302ac4101 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Sun, 5 Apr 2026 22:52:17 +0800 Subject: [PATCH 27/28] chore(helm): bump helm to 4.1.3 --- rootfs/Dockerfile | 4 ++-- rootfs/Dockerfile.test | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index d2661c6..85d3134 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -5,8 +5,8 @@ ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/workspace \ PYTHON_VERSION="3.14" \ - HELM_VERSION="3.19.0" \ - KUBECTL_VERSION="1.34.1" + HELM_VERSION="4.1.3" \ + KUBECTL_VERSION="1.35.3" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test index 4aa8db9..fe48d53 100644 --- a/rootfs/Dockerfile.test +++ b/rootfs/Dockerfile.test @@ -5,8 +5,8 @@ ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/workspace \ PYTHON_VERSION="3.14" \ - HELM_VERSION="3.19.0" \ - KUBECTL_VERSION="1.34.1" + HELM_VERSION="4.1.3" \ + KUBECTL_VERSION="1.35.3" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} From f53626d5a47806df18fdd78f66c8e34006377fbc Mon Sep 17 00:00:00 2001 From: jianxiaoguo Date: Fri, 8 May 2026 11:49:40 +0800 Subject: [PATCH 28/28] chore(charts): config securityContext --- charts/helmbroker/templates/helmbroker-celery-deployment.yaml | 4 ++++ charts/helmbroker/templates/helmbroker-deployment.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/charts/helmbroker/templates/helmbroker-celery-deployment.yaml b/charts/helmbroker/templates/helmbroker-celery-deployment.yaml index b584e72..1b39b78 100644 --- a/charts/helmbroker/templates/helmbroker-celery-deployment.yaml +++ b/charts/helmbroker/templates/helmbroker-celery-deployment.yaml @@ -58,3 +58,7 @@ spec: {{- include "helmbroker.volumeMounts" $ | indent 8 }} {{- end }} {{- include "helmbroker.volumes" . | indent 6 }} + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsUser: 1001 diff --git a/charts/helmbroker/templates/helmbroker-deployment.yaml b/charts/helmbroker/templates/helmbroker-deployment.yaml index fe9a2ed..45cc4cb 100644 --- a/charts/helmbroker/templates/helmbroker-deployment.yaml +++ b/charts/helmbroker/templates/helmbroker-deployment.yaml @@ -78,3 +78,7 @@ spec: {{- include "helmbroker.envs" . | indent 8 }} {{- include "helmbroker.volumeMounts" . | indent 8 }} {{- include "helmbroker.volumes" . | indent 6 }} + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsUser: 1001