chore(controller): use valkey replace redis

duanhongyi · duanhongyi · commit 679c08f0d718 · 2024-11-26T22:49:00.000+08:00
diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
@@ -61,6 +61,21 @@ env:
     secretKeyRef:
       name: builder-key-auth
       key: builder-key
+{{- if (.Values.valkeyUrl) }}
+- name: DRYCC_VALKEY_URL
+  valueFrom:
+    secretKeyRef:
+      name: controller-creds
+      key: valkey-url
+{{- else if eq .Values.global.valkeyLocation "on-cluster"  }}
+- name: VALKEY_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      name: valkey-creds
+      key: password
+- name: DRYCC_VALKEY_URL
+  value: "redis://:$(VALKEY_PASSWORD)@drycc-valkey.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:16379/0"
+{{- end }}
 {{- if (.Values.databaseReplicaUrl) }}
 - name: DRYCC_DATABASE_REPLICA_URL
   valueFrom:
@@ -102,23 +117,13 @@ env:
   valueFrom:
     fieldRef:
       fieldPath: metadata.namespace
-- name: DRYCC_REDIS_ADDRS
-  valueFrom:
-    secretKeyRef:
-      name: redis-creds
-      key: addrs
-- name: DRYCC_REDIS_PASSWORD
-  valueFrom:
-    secretKeyRef:
-      name: redis-creds
-      key: password
-{{- if eq .Values.global.rabbitmqLocation "off-cluster" }}
+{{- if (.Values.prometheusUrl) }}
 - name: "DRYCC_PROMETHEUS_URL"
   valueFrom:
     secretKeyRef:
-      name: prometheus-creds
-      key: url
-{{- else }}
+      name: controller-creds
+      key: prometheus-url
+{{- else if eq .Values.global.prometheusLocation "on-cluster" }}
 - name: "DRYCC_PROMETHEUS_USERNAME"
   valueFrom:
     secretKeyRef:
@@ -132,26 +137,6 @@ env:
 - name: "DRYCC_PROMETHEUS_URL"
   value: "http://$(DRYCC_PROMETHEUS_USERNAME):$(DRYCC_PROMETHEUS_PASSWORD)@drycc-prometheus.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:9090"
 {{- end }}
-{{- if (.Values.rabbitmqUrl) }}
-- name: DRYCC_RABBITMQ_URL
-  valueFrom:
-    secretKeyRef:
-      name: controller-creds
-      key: rabbitmq-url
-{{- else if eq .Values.global.rabbitmqLocation "on-cluster" }}
-- name: "DRYCC_RABBITMQ_USERNAME"
-  valueFrom:
-    secretKeyRef:
-      name: rabbitmq-creds
-      key: username
-- name: "DRYCC_RABBITMQ_PASSWORD"
-  valueFrom:
-    secretKeyRef:
-      name: rabbitmq-creds
-      key: password
-- name: "DRYCC_RABBITMQ_URL"
-  value: "amqp://$(DRYCC_RABBITMQ_USERNAME):$(DRYCC_RABBITMQ_PASSWORD)@drycc-rabbitmq.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:5672/controller"
-{{- end }}
 {{- if eq .Values.global.passportLocation "on-cluster"}}
 - name: "DRYCC_PASSPORT_URL"
 {{- if .Values.global.certManagerEnabled }}
diff --git a/charts/controller/templates/controller-api-deployment.yaml b/charts/controller/templates/controller-api-deployment.yaml
@@ -34,9 +34,7 @@ spec:
         - netcat
         - -v
         - -u
-        - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
-        - -a
-        - $(DRYCC_REDIS_ADDRS)
+        - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
         {{- include "controller.envs" . | indent 8 }}
       containers:
       - name: drycc-controller
diff --git a/charts/controller/templates/controller-celery-deloyment.yaml b/charts/controller/templates/controller-celery-deloyment.yaml
@@ -34,9 +34,9 @@ spec:
           - netcat
           - -v
           - -u
-          - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
+          - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
           - -a
-          - $(DRYCC_REDIS_ADDRS),$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
+          - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
         {{- include "controller.envs" . | indent 8 }}
       containers:
       - name: drycc-controller-celery
diff --git a/charts/controller/templates/controller-cronjob-daily.yaml b/charts/controller/templates/controller-cronjob-daily.yaml
@@ -29,9 +29,9 @@ spec:
               - netcat
               - -v
               - -u
-              - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
+              - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
               - -a
-              - $(DRYCC_REDIS_ADDRS),$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
+              - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
             {{- include "controller.envs" . | indent 12 }}
           containers:
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
diff --git a/charts/controller/templates/controller-cronjob-hourly.yaml b/charts/controller/templates/controller-cronjob-hourly.yaml
@@ -29,9 +29,9 @@ spec:
               - netcat
               - -v
               - -u
-              - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
+              - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
               - -a
-              - $(DRYCC_REDIS_ADDRS),$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
+              - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
             {{- include "controller.envs" . | indent 12 }}
           containers:
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
diff --git a/charts/controller/templates/controller-job-upgrade.yaml b/charts/controller/templates/controller-job-upgrade.yaml
@@ -18,9 +18,7 @@ spec:
         - netcat
         - -v
         - -u
-        - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
-        - -a
-        - $(DRYCC_REDIS_ADDRS)
+        - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
         {{- include "controller.envs" . | indent 8 }}
       containers:
       - name: drycc-controller-job-upgrade
diff --git a/charts/controller/templates/controller-mutate-deloyment.yaml b/charts/controller/templates/controller-mutate-deloyment.yaml
@@ -24,9 +24,9 @@ spec:
         - netcat
         - -v
         - -u
-        - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
+        - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
         - -a
-        - $(DRYCC_REDIS_ADDRS),$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
+        - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
         {{- include "controller.envs" . | indent 8 }}
       containers:
       - name: drycc-controller
diff --git a/charts/controller/templates/controller-secret-creds.yaml b/charts/controller/templates/controller-secret-creds.yaml
@@ -5,14 +5,17 @@ metadata:
   labels:
     heritage: drycc
 data:
+  {{- if (.Values.valkeyUrl) }}
+  valkey-url: {{ .Values.valkeyUrl | b64enc }}
+  {{- end }}
   {{- if (.Values.databaseUrl) }}
   database-url: {{ .Values.databaseUrl | b64enc }}
   {{- end }}
   {{- if (.Values.databaseReplicaUrl) }}
   database-replica-url: {{ .Values.databaseReplicaUrl | b64enc }}
   {{- end }}
-  {{- if (.Values.rabbitmqUrl) }}
-  rabbitmq-url: {{ .Values.rabbitmqUrl | b64enc }}
+  {{- if (.Values.prometheusUrl) }}
+  prometheus-url: {{ .Values.prometheusUrl | b64enc }}
   {{- end }}
   {{- if (.Values.passportUrl) }}
   passport-url: {{ .Values.passportUrl | b64enc }}
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
@@ -54,6 +54,8 @@ mutateReplicas: 1
 # Forbid: The cron job does not allow concurrent runs; if it is time for a new job run and the previous job run hasn't finished yet, the cron job skips the new job run
 # Replace: If it is time for a new job run and the previous job run hasn't finished yet, the cron job replaces the currently running job run with a new job run
 concurrencyPolicy: "Replace"
+## valkeyUrl is will no longer use the built-in valkey component
+valkeyUrl: ""
 # databaseUrl and databaseReplicaUrl are will no longer use the built-in database component
 databaseUrl: ""
 databaseReplicaUrl: ""
@@ -66,9 +68,6 @@ workflowManagerUrl: ""
 workflowManagerAccessKey: ""
 workflowManagerSecretKey: ""
 
-# Configuring this will no longer use the built-in rabbitmq component
-rabbitmqUrl: ""
-
 # limit specs, plans config
 config:
   metrics: ""
@@ -131,9 +130,6 @@ mutate:
     extraMatchLabels:
       component: "drycc-controller-mutate"
 
-redis:
-  replicas: 1
-
 storage:
   csi:
     storageClassName: drycc-storage
@@ -157,7 +153,6 @@ global:
   #   configured above)
   # - off-cluster: Run PostgreSQL outside the Kubernetes cluster (configure in database section)
   databaseLocation: "on-cluster"
-
   # Set the location of Workflow's Registry
   #
   # Valid values are:
@@ -186,3 +181,9 @@ global:
   # Whether cert_manager is enabled to automatically generate controller certificates
   certManagerEnabled: true
   passportLocation: "on-cluster"
+  # Set the location of Workflow's prometheus cluster
+  #
+  # Valid values are:
+  # - on-cluster: Run prometheus within the Kubernetes cluster
+  # - off-cluster: prometheus is running outside of the cluster and credentials and connection information will be provided.
+  prometheusLocation: "on-cluster"
diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test
@@ -8,8 +8,7 @@ COPY dev_requirements.txt ${DRYCC_HOME_DIR}/dev_requirements.txt
 
 ENV PGDATA="/opt/drycc/postgresql/data" \
   PYTHON_VERSION="3.12" \
-  REDIS_VERSION="7.2.4" \
-  RABBITMQ_VERSION="3.12.13" \
+  VALKEY_VERSION="8.0.1" \
   POSTGRES_VERSION="15.5" \
   GOSU_VERSION="1.17"
 
@@ -18,8 +17,7 @@ RUN buildDeps='gcc rustc cargo  libffi-dev musl-dev openssl'; \
   && curl -SsL https://cli.codecov.io/latest/$([ $(dpkg --print-architecture) == "arm64" ] && echo linux-arm64 || echo linux)/codecov -o /usr/local/bin/codecov \
   && chmod +x /usr/local/bin/codecov \
   && install-stack python $PYTHON_VERSION \
-  && install-stack redis $REDIS_VERSION \
-  && install-stack rabbitmq $RABBITMQ_VERSION \
+  && install-stack valkey $VALKEY_VERSION \
   && install-stack postgresql $POSTGRES_VERSION \
   && install-stack gosu $GOSU_VERSION && . init-stack \
   && python3 -m venv ${DRYCC_HOME_DIR}/.venv \
diff --git a/rootfs/api/settings/celery.py b/rootfs/api/settings/celery.py
@@ -5,7 +5,6 @@
 
 class Config(object):
     # Celery Configuration Options
-    timezone = "Asia/Shanghai"
     enable_utc = True
     task_serializer = 'pickle'
     accept_content = frozenset([
@@ -17,20 +16,28 @@ class Config(object):
     task_track_started = True
     task_time_limit = 30 * 60
     worker_max_tasks_per_child = 200
+    worker_prefetch_multiplier = 1
     result_expires = 24 * 60 * 60
-    broker_url = os.environ.get('DRYCC_RABBITMQ_URL', 'amqp://guest:guest@127.0.0.1:5672/')  # noqa
     cache_backend = 'django-cache'
-    task_default_queue = 'controller.low'
+    task_default_queue = 'controller.middle'
     task_default_exchange = 'controller.priority'
-    task_default_routing_key = 'controller.priority.low'
+    task_default_routing_key = 'controller.priority.middle'
+    broker_transport_options = {"queue_order_strategy": "sorted"}
+    task_create_missing_queues = True
+    task_inherit_parent_priority = True
     broker_connection_retry_on_startup = True
     worker_cancel_long_running_tasks_on_connection_loss = True
 
 
 app = Celery('drycc')
 app.config_from_object(Config())
 app.conf.update(
+    timezone=os.environ.get('TZ', 'UTC'),
     task_routes={
+        'api.tasks.send_measurements': {
+            'queue': 'controller.low',
+            'exchange': 'controller.priority', 'routing_key': 'controller.priority.low',
+        },
         'api.tasks.scale_app': {
             'queue': 'controller.high',
             'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
@@ -51,24 +58,26 @@ class Config(object):
             'queue': 'controller.high',
             'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
         },
-        'api.tasks.send_measurements': {
-            'queue': 'controller.middle',
-            'exchange': 'controller.priority', 'routing_key': 'controller.priority.middle',
-        },
     },
     task_queues=(
         Queue(
             'controller.low', exchange=Exchange('controller.priority', type="direct"),
-            routing_key='controller.priority.low', queue_arguments={'x-queue-type': 'quorum'},
+            routing_key='controller.priority.low',
         ),
         Queue(
             'controller.high', exchange=Exchange('controller.priority', type="direct"),
-            routing_key='controller.priority.high', queue_arguments={'x-queue-type': 'quorum'},
+            routing_key='controller.priority.high',
         ),
         Queue(
             'controller.middle', exchange=Exchange('controller.priority', type="direct"),
-            routing_key='controller.priority.middle', queue_arguments={'x-queue-type': 'quorum'},
+            routing_key='controller.priority.middle',
         ),
     ),
 )
+DRYCC_VALKEY_URL = os.environ.get('DRYCC_VALKEY_URL', 'redis://:@127.0.0.1:6379')
+app.conf.update(
+    broker_url=DRYCC_VALKEY_URL,
+    result_backend=DRYCC_VALKEY_URL,
+    broker_transport_options={"queue_order_strategy": "sorted", "visibility_timeout": 43200},
+)
 app.autodiscover_tasks()
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
@@ -8,7 +8,6 @@
 import tempfile
 import dj_database_url
 
-
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, os.path.join(BASE_DIR, 'apps_extra'))
 
@@ -438,20 +437,13 @@
 AUTHENTICATION_BACKENDS = ("api.backend.DryccOIDC", ) + AUTHENTICATION_BACKENDS
 DRYCC_CACHE_USER_TIME = int(os.environ.get('DRYCC_CACHE_USER_TIME', 30 * 60))
 
-# Redis Configuration
-DRYCC_REDIS_ADDRS = os.environ.get('DRYCC_REDIS_ADDRS', '127.0.0.1:6379').split(",")
-DRYCC_REDIS_PASSWORD = os.environ.get('DRYCC_REDIS_PASSWORD', '')
-
-# Cache Configuration
+# Cache Valkey Configuration
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": [
-            'redis://:{}@{}'.format(
-                DRYCC_REDIS_PASSWORD, DRYCC_REDIS_ADDR) for DRYCC_REDIS_ADDR in DRYCC_REDIS_ADDRS
-        ],
+        "LOCATION": os.environ.get('DRYCC_VALKEY_URL', 'redis://:@127.0.0.1:6379'),
         "OPTIONS": {
-            "CLIENT_CLASS": "django_redis.client.ShardClient",
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
         }
     }
 }
diff --git a/rootfs/api/tests/admissions/service_binding_binding_request_in_flight.json b/rootfs/api/tests/admissions/service_binding_binding_request_in_flight.json
@@ -25,7 +25,7 @@
 			"resource": "servicebindings"
 		},
 		"requestSubResource": "status",
-		"name": "redis-t2",
+		"name": "valkey-t2",
 		"namespace": "myapp",
 		"operation": "UPDATE",
 		"userInfo": {
@@ -116,17 +116,17 @@
 						"time": "2024-03-25T01:48:21Z"
 					}
 				],
-				"name": "redis-t2",
+				"name": "valkey-t2",
 				"namespace": "myapp",
 				"resourceVersion": "74043277",
 				"uid": "292b0ef5-947f-42a8-adfb-909e99d26906"
 			},
 			"spec": {
 				"externalID": "57414c7d-0931-4310-8ed6-d8368ac0d3de",
 				"instanceRef": {
-					"name": "redis-t2"
+					"name": "valkey-t2"
 				},
-				"secretName": "redis-t2",
+				"secretName": "valkey-t2",
 				"userInfo": {
 					"extra": {
 						"authentication.kubernetes.io/pod-name": [
@@ -238,17 +238,17 @@
 						"time": "2024-03-25T01:48:21Z"
 					}
 				],
-				"name": "redis-t2",
+				"name": "valkey-t2",
 				"namespace": "myapp",
 				"resourceVersion": "74043277",
 				"uid": "292b0ef5-947f-42a8-adfb-909e99d26906"
 			},
 			"spec": {
 				"externalID": "57414c7d-0931-4310-8ed6-d8368ac0d3de",
 				"instanceRef": {
-					"name": "redis-t2"
+					"name": "valkey-t2"
 				},
-				"secretName": "redis-t2",
+				"secretName": "valkey-t2",
 				"userInfo": {
 					"extra": {
 						"authentication.kubernetes.io/pod-name": [
diff --git a/rootfs/api/tests/admissions/service_binding_ready.json b/rootfs/api/tests/admissions/service_binding_ready.json
diff --git a/rootfs/api/tests/admissions/service_instance_provisioning.json b/rootfs/api/tests/admissions/service_instance_provisioning.json
diff --git a/rootfs/api/tests/admissions/service_instance_ready.json b/rootfs/api/tests/admissions/service_instance_ready.json
diff --git a/rootfs/api/tests/test_admissions.py b/rootfs/api/tests/test_admissions.py
diff --git a/rootfs/bin/test-unit b/rootfs/bin/test-unit
diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
