diff --git a/README.md b/README.md index 28cb9d5..3f4fdb5 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,8 @@ Different components use different db, as follows: * `controller` use db 0 * `passport` use db 1 -* `fluentbit` and `logger` use db 2 +* `grafana` use db 2 +* `builder` use db 3 * `manager` use db 10 * `helmbroker` use db 11 diff --git a/_test/test.sh b/_test/test.sh index badc3e8..abf1d14 100755 --- a/_test/test.sh +++ b/_test/test.sh @@ -2,12 +2,11 @@ VERSION="$1" -CONTAINER_NETWORK="valkey" CONTAINER_PROXY_NAME="valkey-benchmark-proxy" CONTAINER_MASTER_NAME="valkey-benchmark-master" CONTAINER_SLAVE1_NAME="valkey-benchmark-slave1" CONTAINER_SLAVE2_NAME="valkey-benchmark-slave2" -DRYCC_VALKEY_SENTINEL=$CONTAINER_MASTER_NAME +NETWORK_NAME="valkey-benchmark-net" DRYCC_VALKEY_PASSWORD=123456 function clean_before_exit { @@ -16,90 +15,90 @@ function clean_before_exit { } trap clean_before_exit EXIT +ensure-network() { + # Use a dedicated bridge network so each container gets a unique IP and + # can reach its neighbours. The default rootless networking (pasta) makes + # every container share the host IP, which breaks master/replica/sentinel + # discovery because containers cannot connect to each other by that IP. + if ! podman network exists "$NETWORK_NAME" >/dev/null 2>&1; then + podman network create "$NETWORK_NAME" >/dev/null + fi +} + start-valkey-master() { podman run -d \ --rm \ - --network "$CONTAINER_NETWORK" \ - --ip 192.168.253.10 \ - --add-host="$CONTAINER_MASTER_NAME:192.168.253.10" \ - --add-host="$CONTAINER_SLAVE1_NAME:192.168.253.11" \ - --add-host="$CONTAINER_SLAVE2_NAME:192.168.253.12" \ + --network "$NETWORK_NAME" \ --env "REDISCLI_AUTH=$DRYCC_VALKEY_PASSWORD" \ - --env "DRYCC_VALKEY_SENTINEL=$DRYCC_VALKEY_SENTINEL" \ --env "DRYCC_VALKEY_PASSWORD=$DRYCC_VALKEY_PASSWORD" \ --name "$CONTAINER_MASTER_NAME" \ "registry.drycc.cc/drycc/valkey:$VERSION" \ - valkey-start server $CONTAINER_MASTER_NAME - podman exec "$CONTAINER_MASTER_NAME" init-stack valkey-start sentinel $CONTAINER_MASTER_NAME & + sleep infinity + + master_ip=$(podman exec "$CONTAINER_MASTER_NAME" hostname -i | tr -d '\r\n') + podman exec --env "DRYCC_VALKEY_SENTINEL=$master_ip" "$CONTAINER_MASTER_NAME" init-stack valkey-start server "$master_ip" & + podman exec --env "DRYCC_VALKEY_SENTINEL=$master_ip" "$CONTAINER_MASTER_NAME" init-stack valkey-start sentinel "$master_ip" & } start-valkey-slave1() { podman run -d \ --rm \ - --network "$CONTAINER_NETWORK" \ - --ip 192.168.253.11 \ - --add-host="$CONTAINER_MASTER_NAME:192.168.253.10" \ - --add-host="$CONTAINER_SLAVE1_NAME:192.168.253.11" \ - --add-host="$CONTAINER_SLAVE2_NAME:192.168.253.12" \ + --network "$NETWORK_NAME" \ --env "REDISCLI_AUTH=$DRYCC_VALKEY_PASSWORD" \ - --env "DRYCC_VALKEY_SENTINEL=$DRYCC_VALKEY_SENTINEL" \ --env "DRYCC_VALKEY_PASSWORD=$DRYCC_VALKEY_PASSWORD" \ --name "$CONTAINER_SLAVE1_NAME" \ "registry.drycc.cc/drycc/valkey:$VERSION" \ - valkey-start server $CONTAINER_SLAVE1_NAME - podman exec "$CONTAINER_SLAVE1_NAME" init-stack valkey-start sentinel $CONTAINER_SLAVE1_NAME & + sleep infinity + + slave1_ip=$(podman exec -it "$CONTAINER_SLAVE1_NAME" hostname -i | tr -d '\r\n') + podman exec --env "DRYCC_VALKEY_SENTINEL=$master_ip" "$CONTAINER_SLAVE1_NAME" init-stack valkey-start server "$slave1_ip" & + podman exec --env "DRYCC_VALKEY_SENTINEL=$master_ip" "$CONTAINER_SLAVE1_NAME" init-stack valkey-start sentinel "$slave1_ip" & } start-valkey-slave2() { podman run -d \ --rm \ - --network "$CONTAINER_NETWORK" \ - --ip 192.168.253.12 \ - --add-host="$CONTAINER_MASTER_NAME:192.168.253.10" \ - --add-host="$CONTAINER_SLAVE1_NAME:192.168.253.11" \ - --add-host="$CONTAINER_SLAVE2_NAME:192.168.253.12" \ + --network "$NETWORK_NAME" \ --env "REDISCLI_AUTH=$DRYCC_VALKEY_PASSWORD" \ - --env "DRYCC_VALKEY_SENTINEL=$DRYCC_VALKEY_SENTINEL" \ --env "DRYCC_VALKEY_PASSWORD=$DRYCC_VALKEY_PASSWORD" \ --name "$CONTAINER_SLAVE2_NAME" \ "registry.drycc.cc/drycc/valkey:$VERSION" \ - valkey-start server $CONTAINER_SLAVE2_NAME - podman exec "$CONTAINER_SLAVE2_NAME" init-stack valkey-start sentinel $CONTAINER_SLAVE2_NAME & + sleep infinity + + slave2_ip=$(podman exec -it "$CONTAINER_SLAVE2_NAME" hostname -i | tr -d '\r\n') + podman exec --env "DRYCC_VALKEY_SENTINEL=$master_ip" "$CONTAINER_SLAVE2_NAME" init-stack valkey-start server "$slave2_ip" & + podman exec --env "DRYCC_VALKEY_SENTINEL=$master_ip" "$CONTAINER_SLAVE2_NAME" init-stack valkey-start sentinel "$slave2_ip" & } start-valkey-proxy() { podman run -d \ --rm \ - --network "$CONTAINER_NETWORK" \ - --ip 192.168.253.13 \ - --add-host="$CONTAINER_MASTER_NAME:192.168.253.10" \ - --add-host="$CONTAINER_SLAVE1_NAME:192.168.253.11" \ - --add-host="$CONTAINER_SLAVE2_NAME:192.168.253.12" \ + --network "$NETWORK_NAME" \ --env "REDISCLI_AUTH=$DRYCC_VALKEY_PASSWORD" \ - --env "DRYCC_VALKEY_SENTINEL=$DRYCC_VALKEY_SENTINEL" \ --env "DRYCC_VALKEY_PASSWORD=$DRYCC_VALKEY_PASSWORD" \ --name "$CONTAINER_PROXY_NAME" \ "registry.drycc.cc/drycc/valkey:$VERSION" \ - valkey-start proxy + sleep infinity + + podman exec --env "DRYCC_VALKEY_SENTINEL=$master_ip" "$CONTAINER_PROXY_NAME" init-stack valkey-start proxy & } clean-valkey() { { - podman stop -i "$CONTAINER_PROXY_NAME" - podman stop -i "$CONTAINER_SLAVE1_NAME" - podman stop -i "$CONTAINER_SLAVE2_NAME" - podman stop -i "$CONTAINER_MASTER_NAME" - podman network rm -f "$CONTAINER_NETWORK" - } 2>>/dev/null + podman kill "$CONTAINER_PROXY_NAME" + podman kill "$CONTAINER_SLAVE1_NAME" + podman kill "$CONTAINER_SLAVE2_NAME" + podman kill "$CONTAINER_MASTER_NAME" + podman network rm -f "$NETWORK_NAME" + } >>/dev/null 2>&1 } clean-valkey -podman network create --subnet=192.168.253.0/24 "$CONTAINER_NETWORK" +ensure-network start-valkey-master start-valkey-slave1 start-valkey-slave2 start-valkey-proxy - echo "run valkey proxy benchmark..." podman exec "$CONTAINER_PROXY_NAME" init-stack valkey-benchmark -p 16379 -a $DRYCC_VALKEY_PASSWORD diff --git a/charts/valkey/templates/valkey-secret-creds.yaml b/charts/valkey/templates/valkey-secret-creds.yaml index b3f0ea0..33c81d2 100644 --- a/charts/valkey/templates/valkey-secret-creds.yaml +++ b/charts/valkey/templates/valkey-secret-creds.yaml @@ -1,4 +1,3 @@ -{{- if eq .Values.global.valkeyLocation "on-cluster" }} apiVersion: v1 kind: Secret metadata: @@ -8,4 +7,3 @@ metadata: heritage: drycc data: password: {{ include "common.secrets.lookup" (dict "secret" "valkey-creds" "key" "password" "defaultValue" (randAlphaNum 32) "context" $) }} -{{- end }} diff --git a/charts/valkey/templates/valkey-statefulset.yaml b/charts/valkey/templates/valkey-statefulset.yaml index 869ffa5..be223df 100644 --- a/charts/valkey/templates/valkey-statefulset.yaml +++ b/charts/valkey/templates/valkey-statefulset.yaml @@ -1,4 +1,3 @@ -{{- if eq .Values.global.valkeyLocation "on-cluster" }} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -23,7 +22,7 @@ spec: podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }} nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values ) | nindent 10 }} containers: - - name: proxy + - name: drycc-valkey-proxy image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/valkey:{{ .Values.imageTag }} imagePullPolicy: {{ .Values.imagePullPolicy }} {{- if .Values.diagnosticMode.enabled }} @@ -41,9 +40,13 @@ spec: name: valkey-creds key: password - name: DRYCC_VALKEY_SENTINEL - value: {{ printf "drycc-valkey.%s.svc.%s" $.Release.Namespace $.Values.global.clusterDomain }} + value: drycc-valkey ports: - containerPort: 16379 + {{- with index .Values "proxy" "resources" }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} startupProbe: initialDelaySeconds: 10 periodSeconds: 10 @@ -68,19 +71,9 @@ spec: failureThreshold: 5 tcpSocket: port: 16379 - - name: server + - name: drycc-valkey-server image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/valkey:{{ .Values.imageTag }} imagePullPolicy: {{ .Values.imagePullPolicy }} - {{- if or (.Values.limitsCpu) (.Values.limitsMemory)}} - resources: - limits: - {{- if (.Values.limitsCpu) }} - cpu: {{.Values.limitsCpu}} - {{- end}} - {{- if (.Values.limitsMemory) }} - memory: {{.Values.limitsMemory}} - {{- end}} - {{- end}} {{- if .Values.diagnosticMode.enabled }} command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }} args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }} @@ -88,17 +81,21 @@ spec: args: - valkey-start - server - - $(POD_NAME).{{ printf "drycc-valkey.%s.svc.%s" $.Release.Namespace $.Values.global.clusterDomain }} + - $(POD_NAME).drycc-valkey {{- end }} ports: - containerPort: 6379 + {{- with index .Values "server" "resources" }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} env: - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: DRYCC_VALKEY_SENTINEL - value: {{ printf "drycc-valkey.%s.svc.%s" $.Release.Namespace $.Values.global.clusterDomain }} + value: drycc-valkey - name: DRYCC_VALKEY_PASSWORD valueFrom: secretKeyRef: @@ -149,19 +146,9 @@ spec: - /bin/bash - -c - /scripts/prestop-valkey.sh - - name: sentinel + - name: drycc-valkey-sentinel image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/valkey:{{ .Values.imageTag }} imagePullPolicy: {{ .Values.imagePullPolicy }} - {{- if or (.Values.limitsCpu) (.Values.limitsMemory)}} - resources: - limits: - {{- if (.Values.limitsCpu) }} - cpu: {{.Values.limitsCpu}} - {{- end}} - {{- if (.Values.limitsMemory) }} - memory: {{.Values.limitsMemory}} - {{- end}} - {{- end}} {{- if .Values.diagnosticMode.enabled }} command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }} args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }} @@ -169,17 +156,21 @@ spec: args: - valkey-start - sentinel - - $(POD_NAME).{{ printf "drycc-valkey.%s.svc.%s" $.Release.Namespace $.Values.global.clusterDomain }} + - $(POD_NAME).drycc-valkey {{- end }} ports: - containerPort: 26379 + {{- with index .Values "sentinel" "resources" }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} env: - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: DRYCC_VALKEY_SENTINEL - value: {{ printf "drycc-valkey.%s.svc.%s" $.Release.Namespace $.Values.global.clusterDomain }} + value: drycc-valkey - name: DRYCC_VALKEY_PASSWORD valueFrom: secretKeyRef: @@ -230,6 +221,52 @@ spec: - /bin/bash - -c - /scripts/prestop-sentinel.sh + - name: drycc-valkey-metrics + image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/valkey:{{ .Values.imageTag }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }} + {{- else }} + command: + - init-stack + args: + - redis_exporter + {{- end }} + ports: + - containerPort: 9121 + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: valkey-creds + key: password + startupProbe: + initialDelaySeconds: 10 + tcpSocket: + port: 9121 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 15 + successThreshold: 1 + livenessProbe: + initialDelaySeconds: 5 + httpGet: + path: / + port: 9121 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + readinessProbe: + initialDelaySeconds: 5 + httpGet: + path: / + port: 9121 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 securityContext: fsGroup: 1001 runAsGroup: 1001 @@ -251,4 +288,3 @@ spec: requests: storage: {{ .Values.persistence.size | quote }} {{- end }} -{{- end }} diff --git a/charts/valkey/templates/valkey-svc.yaml b/charts/valkey/templates/valkey-svc.yaml index 2300909..c38af8c 100644 --- a/charts/valkey/templates/valkey-svc.yaml +++ b/charts/valkey/templates/valkey-svc.yaml @@ -1,9 +1,11 @@ -{{- if eq .Values.global.valkeyLocation "on-cluster" }} apiVersion: v1 kind: Service metadata: name: drycc-valkey annotations: + prometheus.io/path: /metrics + prometheus.io/port: "9121" + prometheus.io/scrape: "true" {{- with .Values.service.annotations }} {{- toYaml . | nindent 4 }} {{- end }} @@ -21,10 +23,13 @@ spec: port: 6379 targetPort: 6379 protocol: TCP + - name: metrics + port: 9121 + targetPort: 9121 + protocol: TCP - name: sentinel port: 26379 targetPort: 26379 protocol: TCP selector: app: drycc-valkey -{{- end }} diff --git a/charts/valkey/values.yaml b/charts/valkey/values.yaml index 1471d44..bf19ca1 100644 --- a/charts/valkey/values.yaml +++ b/charts/valkey/values.yaml @@ -2,8 +2,6 @@ imageOrg: "drycc" imagePullPolicy: "Always" imageTag: "canary" imageRegistry: "registry.drycc.cc" -# limitsCpu: "100m" -# limitsMemory: "50Mi" ## Enable diagnostic mode ## @@ -39,31 +37,40 @@ podAntiAffinityPreset: # The following parameters are configured only when using an on-cluster Valkey instance replicas: 3 -# The following parameters are configured only when using an off-cluster Valkey instance -addrs: "" # A list of clusters: "127.0.0.1:7001/1,127.0.0.2:7002/1" -password: "valkey password" # "" == no password - # Service service: # Provide any additional service annotations annotations: {} +proxy: + resources: {} + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi + +server: + resources: {} + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi + +sentinel: + resources: {} + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi + # GCP PDs and EBS volumes are supported only persistence: enabled: false # Set to true to enable persistence size: 5Gi storageClass: "" - -global: - # A domain name consists of one or more parts. - # Periods (.) are used to separate these parts. - # Each part must be 1 to 63 characters in length and can contain lowercase letters, digits, and hyphens (-). - # It must start and end with a lowercase letter or digit. - clusterDomain: "cluster.local" - # Set the location of Workflow's valkey instance - # - # Valid values are: - # - on-cluster: Run Valkey within the Kubernetes cluster - # - off-cluster: Run Valkey outside the Kubernetes cluster (configure in valkey section) - valkeyLocation: "on-cluster" - \ No newline at end of file diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index a67a5bc..3de14d7 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -1,11 +1,12 @@ ARG CODENAME FROM registry.drycc.cc/drycc/base:${CODENAME} -ENV DRYCC_UID=1001 \ +ARG DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/data \ - VALKEY_VERSION="8.1.0" \ - VALKEY_SENTINEL_PROXY_VERSION="1.0.2" + VALKEY_VERSION="9.0.4" \ + VALKEY_SENTINEL_PROXY_VERSION="2.0.1" \ + REDIS_EXPORTER_VERSION="1.80.1" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} @@ -16,6 +17,7 @@ COPY bin/valkey-start /bin/valkey-start RUN install-stack valkey ${VALKEY_VERSION} \ && install-stack valkey-sentinel-proxy ${VALKEY_SENTINEL_PROXY_VERSION} \ + && install-stack redis_exporter $REDIS_EXPORTER_VERSION \ && rm -rf \ /usr/share/doc \ /usr/share/man \ @@ -34,4 +36,4 @@ RUN install-stack valkey ${VALKEY_VERSION} \ USER ${DRYCC_UID} WORKDIR ${DRYCC_HOME_DIR} -EXPOSE 6379 26379 +EXPOSE 6379 9121 26379 diff --git a/rootfs/bin/valkey-start b/rootfs/bin/valkey-start index 77618ae..d8204d8 100755 --- a/rootfs/bin/valkey-start +++ b/rootfs/bin/valkey-start @@ -10,7 +10,7 @@ print_usage() { echo "server start valkey server" echo "sentinel start valkey sentinel" echo "" - echo "Such as 'valkey-start server valkey.valkey.svc.cluster.local' to start valkey server." + echo "Such as 'valkey-start server valkey.valkey.svc' to start valkey server." } remove_in_file() { @@ -25,7 +25,23 @@ get_master_info() { eval "$command" } +wait_for_valkey() { + local host="${1:?host is required}" + local port="${2:?port is required}" + local retries="${3:-30}" + log "Waiting for valkey (${host}:${port}) to be ready..." + for ((i = 1; i <= retries; i += 1)); do + if valkey-cli -h "$host" -p "$port" ping 2>/dev/null | grep -q PONG; then + log "Valkey (${host}:${port}) is ready." + return 0 + fi + sleep 2s + done + error "Valkey (${host}:${port}) did not become ready in time" +} + start_valkey_proxy() { + wait_for_valkey "${DRYCC_VALKEY_SENTINEL}" "${SENTINEL_PORT}" exec valkey-sentinel-proxy \ --listen=:16379 \ --master=drycc \ @@ -98,7 +114,7 @@ start_valkey_sentinel() { else printf "\nsentinel monitor drycc %s %s 2" "${announce_ip}" "${SERVER_PORT}" >> ${VALKEY_SENTINEL_CONFIG_FILE} fi - exec valkey-sentinel $VALKEY_SENTINEL_CONFIG_FILE + exec valkey-server $VALKEY_SENTINEL_CONFIG_FILE --sentinel } command="$1"