feat(valkey): add valkey sentinel proxy

Author: duanhongyi

README.md

@@ -3,6 +3,18 @@
 
 Drycc (pronounced DAY-iss) Workflow is an open source Platform as a Service (PaaS) that adds a developer-friendly layer to any [Kubernetes](http://kubernetes.io) cluster, making it easy to deploy and manage applications on your own servers.
 
+## Usage
+
+Different components use different db, as follows:
+
+* `controller` use db 0
+* `passport` use db 1
+* `fluentbit` and `logger` use db 2
+* `manager` use db 10
+* `helmbroker` use db 11
+
+The above are the default configurations for each component.
+
 ## Description
 A Container image for running standalone (not clustered) Valkey on a Kubernetes cluster.
 

_test/test.sh

@@ -3,6 +3,7 @@
 VERSION="$1"
 
 CONTAINER_NETWORK="valkey"
+CONTAINER_PROXY_NAME="valkey-benchmark-proxy"
 CONTAINER_MASTER_NAME="valkey-benchmark-master"
 CONTAINER_SLAVE1_NAME="valkey-benchmark-slave1"
 CONTAINER_SLAVE2_NAME="valkey-benchmark-slave2"
@@ -66,8 +67,25 @@ start-valkey-slave2() {
     podman exec "$CONTAINER_SLAVE2_NAME" init-stack valkey-start sentinel $CONTAINER_SLAVE2_NAME &
 }
 
+start-valkey-proxy() {
+    podman run -d \
+      --rm \
+      --network "$CONTAINER_NETWORK" \
+      --ip 192.168.253.13 \
+      --add-host="$CONTAINER_MASTER_NAME:192.168.253.10" \
+      --add-host="$CONTAINER_SLAVE1_NAME:192.168.253.11" \
+      --add-host="$CONTAINER_SLAVE2_NAME:192.168.253.12" \
+      --env "REDISCLI_AUTH=$DRYCC_VALKEY_PASSWORD" \
+      --env "DRYCC_VALKEY_SENTINEL=$DRYCC_VALKEY_SENTINEL" \
+      --env "DRYCC_VALKEY_PASSWORD=$DRYCC_VALKEY_PASSWORD" \
+      --name "$CONTAINER_PROXY_NAME" \
+      "registry.drycc.cc/drycc/valkey:$VERSION" \
+      valkey-start proxy
+}
+
 clean-valkey() {
     {
+        podman stop -i "$CONTAINER_PROXY_NAME"
         podman stop -i "$CONTAINER_SLAVE1_NAME"
         podman stop -i "$CONTAINER_SLAVE2_NAME"
         podman stop -i "$CONTAINER_MASTER_NAME"
@@ -80,8 +98,12 @@ podman network create --subnet=192.168.253.0/24 "$CONTAINER_NETWORK"
 start-valkey-master
 start-valkey-slave1
 start-valkey-slave2
+start-valkey-proxy
+
+echo "run valkey proxy benchmark..."
+podman exec "$CONTAINER_PROXY_NAME" init-stack valkey-benchmark -p 16379 -a $DRYCC_VALKEY_PASSWORD
 
-echo "run valkey benchmark..."
+echo "run valkey master benchmark..."
 podman exec "$CONTAINER_MASTER_NAME" init-stack valkey-benchmark -a $DRYCC_VALKEY_PASSWORD
 
 echo "check slave all keys..."

charts/valkey/templates/valkey-statefulset.yaml

@@ -23,6 +23,51 @@ spec:
         podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
         nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values ) | nindent 10 }}
       containers:
+      - name: proxy
+        image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/valkey:{{ .Values.imageTag }}
+        imagePullPolicy: {{ .Values.imagePullPolicy }}
+        {{- if .Values.diagnosticMode.enabled }}
+        command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }}
+        args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }}
+        {{- else }}
+        args:
+        - valkey-start
+        - proxy
+        {{- end }}
+        env:
+        - name: DRYCC_VALKEY_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: valkey-creds
+              key: password
+        - name: DRYCC_VALKEY_SENTINEL
+          value: {{ printf "drycc-valkey.%s.svc.%s" $.Release.Namespace $.Values.global.clusterDomain }}
+        ports:
+        - containerPort: 16379
+        startupProbe:
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 22
+          tcpSocket:
+            port: 16379
+        livenessProbe:
+          initialDelaySeconds: 20
+          periodSeconds: 5
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 5
+          tcpSocket:
+            port: 16379
+        readinessProbe:
+          initialDelaySeconds: 20
+          periodSeconds: 5
+          timeoutSeconds: 1
+          successThreshold: 1
+          failureThreshold: 5
+          tcpSocket:
+            port: 16379
       - name: server
         image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/valkey:{{ .Values.imageTag }}
         imagePullPolicy: {{ .Values.imagePullPolicy }}

charts/valkey/templates/valkey-svc.yaml

@@ -13,6 +13,10 @@ spec:
   clusterIP: None
   publishNotReadyAddresses: true
   ports:
+  - name: proxy
+    port: 16379
+    targetPort: 16379
+    protocol: TCP
   - name: server
     port: 6379
     targetPort: 6379

rootfs/Dockerfile

@@ -4,7 +4,8 @@ FROM registry.drycc.cc/drycc/base:${CODENAME}
 ENV DRYCC_UID=1001 \
   DRYCC_GID=1001 \
   DRYCC_HOME_DIR=/data \
-  VALKEY_VERSION="8.0.1"
+  VALKEY_VERSION="8.0.1" \
+  VALKEY_SENTINEL_PROXY_VERSION="1.0.1"
 
 RUN groupadd drycc --gid ${DRYCC_GID} \
   && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR}
@@ -14,6 +15,7 @@ COPY etc/valkey /etc/valkey
 COPY bin/valkey-start /bin/valkey-start
 
 RUN install-stack valkey ${VALKEY_VERSION} \
+  && install-stack valkey-sentinel-proxy ${VALKEY_SENTINEL_PROXY_VERSION} \
   && rm -rf \
       /usr/share/doc \
       /usr/share/man \

rootfs/bin/valkey-start

@@ -6,10 +6,11 @@
 print_usage() {
     echo "Valid commands for valkey-start:"
     echo ""
+    echo "proxy <announce-host>               start valkey proxy"
     echo "server <announce-host>              start valkey server"
     echo "sentinel <announce-host>            start valkey sentinel"
     echo ""
-    echo "Such as 'weed-start server valkey.valkey.svc.cluster.local' to start valkey server."
+    echo "Such as 'valkey-start server valkey.valkey.svc.cluster.local' to start valkey server."
 }
 
 remove_in_file() {
@@ -24,6 +25,15 @@ get_master_info() {
     eval "$command"
 }
 
+start_valkey_proxy() {
+    exec valkey-sentinel-proxy \
+        --listen=:16379 \
+        --master=drycc \
+        --max-procs=4 \
+        --sentinel-addr="${DRYCC_VALKEY_SENTINEL}":26379 \
+        --sentinel-pass="${DRYCC_VALKEY_PASSWORD}"
+}
+
 start_valkey_server() {
     announce_ip="$1"
     VALKEY_CONFIG_FILE=/data/server/valkey.conf
@@ -92,7 +102,7 @@ start_valkey_sentinel() {
 }
 
 command="$1"
-if [[ ${command} == "server" || ${command} == "sentinel" ]]; then
+if [[ ${command} == "proxy" || ${command} == "server" || ${command} == "sentinel" ]]; then
     "start_valkey_$command" "$2"
 else
     print_usage