Skip to content

Commit bd4a1b8

Browse files
duanhongyiduanhongyi
committed
feat(valkey): add sentinel
1 parent 23e00da commit bd4a1b8

19 files changed

Lines changed: 566 additions & 41 deletions

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ PLATFORM ?= linux/amd64,linux/arm64
99

1010
include versioning.mk
1111

12-
SHELL_SCRIPTS = $(wildcard _scripts/*.sh) rootfs/bin/boot
12+
SHELL_SCRIPTS = $(wildcard _test/*.sh) rootfs/bin/valkey-start rootfs/scripts/*
1313

1414
# The following variables describe the containerized development environment
1515
# and other build options

‎_test/test.sh‎

Lines changed: 120 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,136 @@
11
#!/usr/bin/env bash
22

33
VERSION="$1"
4-
CONTAINER_NAME="valkey-benchmark"
4+
5+
CONTAINER_NETWORK="valkey"
6+
CONTAINER_MASTER_NAME="valkey-benchmark-master"
7+
CONTAINER_SLAVE1_NAME="valkey-benchmark-slave1"
8+
CONTAINER_SLAVE2_NAME="valkey-benchmark-slave2"
9+
DRYCC_VALKEY_SENTINEL=$CONTAINER_MASTER_NAME
10+
DRYCC_VALKEY_PASSWORD=123456
511

612
function clean_before_exit {
713
# delay before exiting, so stdout/stderr flushes through the logging system
814
clean-valkey
915
}
1016
trap clean_before_exit EXIT
1117

12-
start-valkey() {
13-
podman run -d --name "$CONTAINER_NAME" --rm registry.drycc.cc/drycc/valkey:$VERSION
18+
start-valkey-master() {
19+
podman run -d \
20+
--rm \
21+
--network "$CONTAINER_NETWORK" \
22+
--ip 192.168.253.10 \
23+
--add-host="$CONTAINER_MASTER_NAME:192.168.253.10" \
24+
--add-host="$CONTAINER_SLAVE1_NAME:192.168.253.11" \
25+
--add-host="$CONTAINER_SLAVE2_NAME:192.168.253.12" \
26+
--env "REDISCLI_AUTH=$DRYCC_VALKEY_PASSWORD" \
27+
--env "DRYCC_VALKEY_SENTINEL=$DRYCC_VALKEY_SENTINEL" \
28+
--env "DRYCC_VALKEY_PASSWORD=$DRYCC_VALKEY_PASSWORD" \
29+
--name "$CONTAINER_MASTER_NAME" \
30+
"registry.drycc.cc/drycc/valkey:$VERSION" \
31+
valkey-start server $CONTAINER_MASTER_NAME
32+
podman exec "$CONTAINER_MASTER_NAME" init-stack valkey-start sentinel $CONTAINER_MASTER_NAME &
33+
}
34+
35+
start-valkey-slave1() {
36+
podman run -d \
37+
--rm \
38+
--network "$CONTAINER_NETWORK" \
39+
--ip 192.168.253.11 \
40+
--add-host="$CONTAINER_MASTER_NAME:192.168.253.10" \
41+
--add-host="$CONTAINER_SLAVE1_NAME:192.168.253.11" \
42+
--add-host="$CONTAINER_SLAVE2_NAME:192.168.253.12" \
43+
--env "REDISCLI_AUTH=$DRYCC_VALKEY_PASSWORD" \
44+
--env "DRYCC_VALKEY_SENTINEL=$DRYCC_VALKEY_SENTINEL" \
45+
--env "DRYCC_VALKEY_PASSWORD=$DRYCC_VALKEY_PASSWORD" \
46+
--name "$CONTAINER_SLAVE1_NAME" \
47+
"registry.drycc.cc/drycc/valkey:$VERSION" \
48+
valkey-start server $CONTAINER_SLAVE1_NAME
49+
podman exec "$CONTAINER_SLAVE1_NAME" init-stack valkey-start sentinel $CONTAINER_SLAVE1_NAME &
50+
}
51+
52+
start-valkey-slave2() {
53+
podman run -d \
54+
--rm \
55+
--network "$CONTAINER_NETWORK" \
56+
--ip 192.168.253.12 \
57+
--add-host="$CONTAINER_MASTER_NAME:192.168.253.10" \
58+
--add-host="$CONTAINER_SLAVE1_NAME:192.168.253.11" \
59+
--add-host="$CONTAINER_SLAVE2_NAME:192.168.253.12" \
60+
--env "REDISCLI_AUTH=$DRYCC_VALKEY_PASSWORD" \
61+
--env "DRYCC_VALKEY_SENTINEL=$DRYCC_VALKEY_SENTINEL" \
62+
--env "DRYCC_VALKEY_PASSWORD=$DRYCC_VALKEY_PASSWORD" \
63+
--name "$CONTAINER_SLAVE2_NAME" \
64+
"registry.drycc.cc/drycc/valkey:$VERSION" \
65+
valkey-start server $CONTAINER_SLAVE2_NAME
66+
podman exec "$CONTAINER_SLAVE2_NAME" init-stack valkey-start sentinel $CONTAINER_SLAVE2_NAME &
1467
}
1568

1669
clean-valkey() {
17-
podman rm -f "$CONTAINER_NAME"
70+
{
71+
podman stop -i "$CONTAINER_SLAVE1_NAME"
72+
podman stop -i "$CONTAINER_SLAVE2_NAME"
73+
podman stop -i "$CONTAINER_MASTER_NAME"
74+
podman network rm -f "$CONTAINER_NETWORK"
75+
} 2>>/dev/null
1876
}
1977

2078
clean-valkey
21-
start-valkey
22-
podman exec "$CONTAINER_NAME" /opt/drycc/valkey/bin/valkey-benchmark
79+
podman network create --subnet=192.168.253.0/24 "$CONTAINER_NETWORK"
80+
start-valkey-master
81+
start-valkey-slave1
82+
start-valkey-slave2
83+
84+
echo "run valkey benchmark..."
85+
podman exec "$CONTAINER_MASTER_NAME" init-stack valkey-benchmark -a $DRYCC_VALKEY_PASSWORD
86+
87+
echo "check slave all keys..."
88+
KEYS=$(podman exec "$CONTAINER_MASTER_NAME" bash -c 'init-stack valkey-cli KEYS "*"')
89+
if [[ "${KEYS}" == "" ]]; then
90+
echo "error: there is no data from the database"
91+
exit 1
92+
fi
93+
94+
echo "check sentinel $CONTAINER_MASTER_NAME get master..."
95+
MASTER=$(podman exec "$CONTAINER_MASTER_NAME" init-stack valkey-cli -p 26379 sentinel get-master-addr-by-name drycc)
96+
if [[ "${MASTER}" == "" ]]; then
97+
echo "error: unable to obtain master information"
98+
exit 1
99+
fi
100+
101+
echo "check sentinel $CONTAINER_SLAVE1_NAME get master..."
102+
MASTER=$(podman exec "$CONTAINER_SLAVE1_NAME" init-stack valkey-cli -p 26379 sentinel get-master-addr-by-name drycc)
103+
if [[ "${MASTER}" == "" ]]; then
104+
echo "error: unable to obtain master information"
105+
exit 1
106+
fi
107+
108+
echo "check sentinel $CONTAINER_SLAVE2_NAME get master..."
109+
MASTER=$(podman exec "$CONTAINER_SLAVE2_NAME" init-stack valkey-cli -p 26379 sentinel get-master-addr-by-name drycc)
110+
if [[ "${MASTER}" == "" ]]; then
111+
echo "error: unable to obtain master information"
112+
exit 1
113+
fi
114+
115+
echo "check sentinel $CONTAINER_MASTER_NAME get slaves..."
116+
SLAVES=$(podman exec "$CONTAINER_MASTER_NAME" init-stack valkey-cli -p 26379 sentinel replicas drycc)
117+
if [[ "${SLAVES}" == "" ]]; then
118+
echo "error: unable to obtain slaves information"
119+
exit 1
120+
fi
121+
122+
echo "check sentinel $CONTAINER_SLAVE1_NAME get slaves..."
123+
SLAVES=$(podman exec "$CONTAINER_SLAVE1_NAME" init-stack valkey-cli -p 26379 sentinel replicas drycc)
124+
if [[ "${SLAVES}" == "" ]]; then
125+
echo "error: unable to obtain slaves information"
126+
exit 1
127+
fi
128+
129+
echo "check sentinel $CONTAINER_SLAVE2_NAME get slaves..."
130+
SLAVES=$(podman exec "$CONTAINER_SLAVE2_NAME" init-stack valkey-cli -p 26379 sentinel replicas drycc)
131+
if [[ "${SLAVES}" == "" ]]; then
132+
echo "error: unable to obtain slaves information"
133+
exit 1
134+
fi
135+
136+
echo "all test ok..."

‎charts/valkey/Chart.lock‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
dependencies:
2+
- name: common
3+
repository: oci://registry.drycc.cc/charts
4+
version: 1.1.3
5+
digest: sha256:4928b76b9f4927f43702c2617b139b2fe7298055b9b6d0c363a7a94530985595
6+
generated: "2024-11-20T16:35:58.537426209+08:00"

‎charts/valkey/charts/common-1.1.3.tgz‎

10.4 KB
Binary file not shown.

‎charts/valkey/templates/valkey-secret-creds.yaml‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,11 @@
1+
{{- if eq .Values.global.valkeyLocation "on-cluster" }}
12
apiVersion: v1
23
kind: Secret
34
metadata:
45
name: valkey-creds
56
labels:
67
app: drycc-valkey
78
heritage: drycc
8-
data:
9-
{{ if eq .Values.global.valkeyLocation "on-cluster"}}
10-
{{- $valkeyAddrs := (list ) }}
11-
{{- $valkeyNodeCount := .Values.replicas | int }}
12-
{{- range $i := until $valkeyNodeCount }}
13-
{{- $valkeyAddrs = (append $valkeyAddrs (printf "drycc-valkey-%d.drycc-valkey.%s.svc.%s:6379" $i $.Release.Namespace $.Values.global.clusterDomain)) }}
14-
{{- end -}}
15-
addrs: {{ join "," $valkeyAddrs | b64enc }}
9+
data:
1610
password: {{ include "common.secrets.lookup" (dict "secret" "valkey-creds" "key" "password" "defaultValue" (randAlphaNum 32) "context" $) }}
17-
{{ else if eq .Values.global.valkeyLocation "off-cluster"}}
18-
addrs: {{ .Values.addrs | b64enc }}
19-
password: {{ .Values.password | b64enc }}
20-
{{ end }}
11+
{{- end }}

‎charts/valkey/templates/valkey-statefulset.yaml‎

Lines changed: 131 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ spec:
2323
podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
2424
nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values ) | nindent 10 }}
2525
containers:
26-
- name: drycc-valkey
26+
- name: server
2727
image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/valkey:{{ .Values.imageTag }}
2828
imagePullPolicy: {{ .Values.imagePullPolicy }}
2929
{{- if or (.Values.limitsCpu) (.Values.limitsMemory)}}
@@ -41,23 +41,150 @@ spec:
4141
args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }}
4242
{{- else }}
4343
args:
44-
- "/bin/boot"
45-
- "--port"
46-
- "6379"
44+
- valkey-start
45+
- server
46+
- $(POD_NAME).{{ printf "drycc-valkey.%s.svc.%s" $.Release.Namespace $.Values.global.clusterDomain }}
4747
{{- end }}
4848
ports:
4949
- containerPort: 6379
5050
env:
51+
- name: POD_NAME
52+
valueFrom:
53+
fieldRef:
54+
fieldPath: metadata.name
55+
- name: DRYCC_VALKEY_SENTINEL
56+
value: {{ printf "drycc-valkey.%s.svc.%s" $.Release.Namespace $.Values.global.clusterDomain }}
57+
- name: DRYCC_VALKEY_PASSWORD
58+
valueFrom:
59+
secretKeyRef:
60+
name: valkey-creds
61+
key: password
62+
startupProbe:
63+
initialDelaySeconds: 10
64+
periodSeconds: 10
65+
timeoutSeconds: 5
66+
successThreshold: 1
67+
failureThreshold: 22
68+
tcpSocket:
69+
port: 6379
70+
livenessProbe:
71+
initialDelaySeconds: 20
72+
periodSeconds: 5
73+
timeoutSeconds: 5
74+
successThreshold: 1
75+
failureThreshold: 5
76+
exec:
77+
command:
78+
- init-stack
79+
- sh
80+
- -c
81+
- /scripts/ping_liveness_local.sh 5
82+
readinessProbe:
83+
initialDelaySeconds: 20
84+
periodSeconds: 5
85+
timeoutSeconds: 1
86+
successThreshold: 1
87+
failureThreshold: 5
88+
exec:
89+
command:
90+
- init-stack
91+
- sh
92+
- -c
93+
- /scripts/ping_readiness_local.sh 1
94+
{{- if .Values.persistence.enabled }}
95+
volumeMounts:
96+
- name: valkey-data
97+
mountPath: /data
98+
{{- end }}
99+
lifecycle:
100+
preStop:
101+
exec:
102+
command:
103+
- init-stack
104+
- /bin/bash
105+
- -c
106+
- /scripts/prestop-valkey.sh
107+
- name: sentinel
108+
image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/valkey:{{ .Values.imageTag }}
109+
imagePullPolicy: {{ .Values.imagePullPolicy }}
110+
{{- if or (.Values.limitsCpu) (.Values.limitsMemory)}}
111+
resources:
112+
limits:
113+
{{- if (.Values.limitsCpu) }}
114+
cpu: {{.Values.limitsCpu}}
115+
{{- end}}
116+
{{- if (.Values.limitsMemory) }}
117+
memory: {{.Values.limitsMemory}}
118+
{{- end}}
119+
{{- end}}
120+
{{- if .Values.diagnosticMode.enabled }}
121+
command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }}
122+
args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }}
123+
{{- else }}
124+
args:
125+
- valkey-start
126+
- sentinel
127+
- $(POD_NAME).{{ printf "drycc-valkey.%s.svc.%s" $.Release.Namespace $.Values.global.clusterDomain }}
128+
{{- end }}
129+
ports:
130+
- containerPort: 26379
131+
env:
132+
- name: POD_NAME
133+
valueFrom:
134+
fieldRef:
135+
fieldPath: metadata.name
136+
- name: DRYCC_VALKEY_SENTINEL
137+
value: {{ printf "drycc-valkey.%s.svc.%s" $.Release.Namespace $.Values.global.clusterDomain }}
51138
- name: DRYCC_VALKEY_PASSWORD
52139
valueFrom:
53140
secretKeyRef:
54141
name: valkey-creds
55142
key: password
143+
startupProbe:
144+
initialDelaySeconds: 10
145+
periodSeconds: 10
146+
timeoutSeconds: 5
147+
successThreshold: 1
148+
failureThreshold: 22
149+
tcpSocket:
150+
port: 26379
151+
livenessProbe:
152+
initialDelaySeconds: 20
153+
periodSeconds: 5
154+
timeoutSeconds: 5
155+
successThreshold: 1
156+
failureThreshold: 5
157+
exec:
158+
command:
159+
- init-stack
160+
- sh
161+
- -c
162+
- /scripts/ping_sentinel.sh 5
163+
readinessProbe:
164+
initialDelaySeconds: 20
165+
periodSeconds: 5
166+
timeoutSeconds: 1
167+
successThreshold: 1
168+
failureThreshold: 5
169+
exec:
170+
command:
171+
- init-stack
172+
- sh
173+
- -c
174+
- /scripts/ping_sentinel.sh 1
56175
{{- if .Values.persistence.enabled }}
57176
volumeMounts:
58177
- name: valkey-data
59178
mountPath: /data
60179
{{- end }}
180+
lifecycle:
181+
preStop:
182+
exec:
183+
command:
184+
- init-stack
185+
- /bin/bash
186+
- -c
187+
- /scripts/prestop-sentinel.sh
61188
securityContext:
62189
fsGroup: 1001
63190
runAsGroup: 1001

‎charts/valkey/templates/valkey-svc.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,16 @@ metadata:
1111
heritage: drycc
1212
spec:
1313
clusterIP: None
14+
publishNotReadyAddresses: true
15+
ports:
16+
- name: server
17+
port: 6379
18+
targetPort: 6379
19+
protocol: TCP
20+
- name: sentinel
21+
port: 26379
22+
targetPort: 26379
23+
protocol: TCP
1424
selector:
1525
app: drycc-valkey
1626
{{- end }}

‎charts/valkey/values.yaml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ podAntiAffinityPreset:
3737
app: "drycc-valkey"
3838

3939
# The following parameters are configured only when using an on-cluster Valkey instance
40-
replicas: 1
40+
replicas: 3
4141

4242
# The following parameters are configured only when using an off-cluster Valkey instance
4343
addrs: "" # A list of clusters: "127.0.0.1:7001/1,127.0.0.2:7002/1"
@@ -55,9 +55,15 @@ persistence:
5555
storageClass: ""
5656

5757
global:
58+
# A domain name consists of one or more parts.
59+
# Periods (.) are used to separate these parts.
60+
# Each part must be 1 to 63 characters in length and can contain lowercase letters, digits, and hyphens (-).
61+
# It must start and end with a lowercase letter or digit.
62+
clusterDomain: "cluster.local"
5863
# Set the location of Workflow's valkey instance
5964
#
6065
# Valid values are:
6166
# - on-cluster: Run Valkey within the Kubernetes cluster
6267
# - off-cluster: Run Valkey outside the Kubernetes cluster (configure in valkey section)
63-
valkeyLocation: "on-cluster"
68+
valkeyLocation: "on-cluster"
69+

0 commit comments

Comments
 (0)