From 43a96716230c9d9667c32fe8bcf7a69d5ff09ab7 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 7 May 2025 18:03:06 +0800 Subject: [PATCH 01/15] chore(charts): use condition --- charts/storage/templates/storage-secret-creds.yaml | 14 +------------- .../storage/templates/storage-service-account.yaml | 2 -- charts/storage/templates/storage-service.yaml | 2 -- charts/storage/templates/storage-statefulset.yaml | 2 -- charts/storage/values.yaml | 6 ------ 5 files changed, 1 insertion(+), 25 deletions(-) diff --git a/charts/storage/templates/storage-secret-creds.yaml b/charts/storage/templates/storage-secret-creds.yaml index 3cdff7e..22f2354 100644 --- a/charts/storage/templates/storage-secret-creds.yaml +++ b/charts/storage/templates/storage-secret-creds.yaml @@ -6,17 +6,5 @@ metadata: heritage: drycc type: Opaque data: - {{- if eq .Values.global.storageLocation "on-cluster"}} - lookup: {{ "path" | b64enc }} - {{- $endpoint := (printf "http://drycc-storage.%s.svc.%s:9000" .Release.Namespace .Values.global.clusterDomain) }} - endpoint: {{ $endpoint | b64enc }} - builder-bucket: {{ "builder" | b64enc }} - registry-bucket: {{ "registry" | b64enc }} - {{- else }} - lookup: {{ .Values.lookup | b64enc }} - endpoint: {{ .Values.endpoint | b64enc }} - builder-bucket: {{ .Values.builderBucket | b64enc }} - registry-bucket: {{ .Values.registryBucket | b64enc }} - {{- end }} accesskey: {{ include "common.secrets.lookup" (dict "secret" "storage-creds" "key" "accesskey" "defaultValue" (include "storage.accesskey" .) "context" $) }} - secretkey: {{ include "common.secrets.lookup" (dict "secret" "storage-creds" "key" "secretkey" "defaultValue" (include "storage.secretkey" .) "context" $) }} \ No newline at end of file + secretkey: {{ include "common.secrets.lookup" (dict "secret" "storage-creds" "key" "secretkey" "defaultValue" (include "storage.secretkey" .) "context" $) }} diff --git a/charts/storage/templates/storage-service-account.yaml b/charts/storage/templates/storage-service-account.yaml index e09c31b..9b17ef9 100644 --- a/charts/storage/templates/storage-service-account.yaml +++ b/charts/storage/templates/storage-service-account.yaml @@ -1,8 +1,6 @@ -{{- if eq .Values.global.storageLocation "on-cluster" }} apiVersion: v1 kind: ServiceAccount metadata: name: drycc-storage labels: heritage: drycc -{{- end }} diff --git a/charts/storage/templates/storage-service.yaml b/charts/storage/templates/storage-service.yaml index b8e651b..a4d0301 100644 --- a/charts/storage/templates/storage-service.yaml +++ b/charts/storage/templates/storage-service.yaml @@ -1,4 +1,3 @@ -{{- if eq .Values.global.storageLocation "on-cluster" }} apiVersion: v1 kind: Service metadata: @@ -21,4 +20,3 @@ spec: port: 9000 - name: console port: 9001 -{{- end }} diff --git a/charts/storage/templates/storage-statefulset.yaml b/charts/storage/templates/storage-statefulset.yaml index f4ccbaf..eb1226d 100644 --- a/charts/storage/templates/storage-statefulset.yaml +++ b/charts/storage/templates/storage-statefulset.yaml @@ -1,4 +1,3 @@ -{{- if eq .Values.global.storageLocation "on-cluster" }} {{- $zoneCount := int .Values.zones }} {{- $driveCount := int .Values.drives }} {{- $replicaCount := int .Values.replicas }} @@ -118,4 +117,3 @@ spec: emptyDir: {} {{- end }} {{- end }} -{{- end }} diff --git a/charts/storage/values.yaml b/charts/storage/values.yaml index db160b0..2ed8de9 100644 --- a/charts/storage/values.yaml +++ b/charts/storage/values.yaml @@ -45,9 +45,3 @@ global: # Each part must be 1 to 63 characters in length and can contain lowercase letters, digits, and hyphens (-). # It must start and end with a lowercase letter or digit. clusterDomain: "cluster.local" - # Set the location of Workflow's Object Storage - # - # Valid values are: - # - on-cluster: Run drycc storage within the Kubernetes cluster - # - off-cluster: Run Storage outside the Kubernetes cluster (Compatible with s3 API) - storageLocation: on-cluster \ No newline at end of file From fb5f448ae4d4bd1c09b96739ed1a8b81bc20852e Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Mon, 12 May 2025 09:24:43 +0800 Subject: [PATCH 02/15] chore(charts): change resources format --- charts/storage/templates/storage-statefulset.yaml | 4 ++++ charts/storage/values.yaml | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/charts/storage/templates/storage-statefulset.yaml b/charts/storage/templates/storage-statefulset.yaml index eb1226d..d80f333 100644 --- a/charts/storage/templates/storage-statefulset.yaml +++ b/charts/storage/templates/storage-statefulset.yaml @@ -52,6 +52,10 @@ spec: - name: console containerPort: 9001 protocol: TCP + {{- with index .Values "resources" }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} livenessProbe: httpGet: path: /minio/health/live diff --git a/charts/storage/values.yaml b/charts/storage/values.yaml index 2ed8de9..2beda1e 100644 --- a/charts/storage/values.yaml +++ b/charts/storage/values.yaml @@ -3,6 +3,14 @@ imagePullPolicy: "Always" imageTag: "canary" imageRegistry: "registry.drycc.cc" +resources: {} + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi + # The following parameters are configured only when using an on-cluster Minio instance nodeAffinityPreset: key: "drycc.cc/node" From 168bdb7d9b7d95aaf2994af28bcd19b998601b35 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 14 May 2025 09:48:17 +0800 Subject: [PATCH 03/15] fix(charts): start minio error --- charts/storage/templates/storage-statefulset.yaml | 14 +++++++++++--- charts/storage/values.yaml | 6 +++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/charts/storage/templates/storage-statefulset.yaml b/charts/storage/templates/storage-statefulset.yaml index d80f333..1f1edb8 100644 --- a/charts/storage/templates/storage-statefulset.yaml +++ b/charts/storage/templates/storage-statefulset.yaml @@ -1,5 +1,5 @@ {{- $zoneCount := int .Values.zones }} -{{- $driveCount := int .Values.drives }} +{{- $driveCount := int .Values.drivesPerNode }} {{- $replicaCount := int .Values.replicas }} apiVersion: apps/v1 kind: StatefulSet @@ -60,9 +60,9 @@ spec: httpGet: path: /minio/health/live port: 9000 - initialDelaySeconds: 90 + initialDelaySeconds: 5 periodSeconds: 5 - timeoutSeconds: 1 + timeoutSeconds: 5 successThreshold: 1 failureThreshold: 5 readinessProbe: @@ -73,6 +73,14 @@ spec: timeoutSeconds: 1 successThreshold: 1 failureThreshold: 5 + startupProbe: + tcpSocket: + port: 9000 + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 args: - minio - server diff --git a/charts/storage/values.yaml b/charts/storage/values.yaml index 2beda1e..dde6228 100644 --- a/charts/storage/values.yaml +++ b/charts/storage/values.yaml @@ -26,12 +26,12 @@ podAntiAffinityPreset: extraMatchLabels: app: "drycc-storage" +# Number of replicas per zone +replicas: 4 # Minio zone count zones: 1 # Device count of per replica -drives: 1 -# Number of replicas per zone -replicas: 4 +drivesPerNode: 1 # Access key and secret key for storage accesskey: "" From 0fda4b74a8149e2ae0999a90d79465369d1e634d Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 14 May 2025 14:01:51 +0800 Subject: [PATCH 04/15] chore(charts): change prometheus path --- charts/storage/templates/storage-service.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/storage/templates/storage-service.yaml b/charts/storage/templates/storage-service.yaml index a4d0301..0467359 100644 --- a/charts/storage/templates/storage-service.yaml +++ b/charts/storage/templates/storage-service.yaml @@ -3,7 +3,7 @@ kind: Service metadata: name: drycc-storage annotations: - prometheus.io/path: /minio/v2/metrics/resource + prometheus.io/path: /minio/v2/metrics/cluster prometheus.io/port: "9000" prometheus.io/scrape: "true" {{- with .Values.service.annotations }} From ca4063ca87fbf35c2e6c8a323243255cd68c1ecc Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 14 May 2025 14:34:17 +0800 Subject: [PATCH 05/15] chore(charts): add prometheus rule support --- charts/storage/templates/storage-service.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/storage/templates/storage-service.yaml b/charts/storage/templates/storage-service.yaml index 0467359..cafa2a5 100644 --- a/charts/storage/templates/storage-service.yaml +++ b/charts/storage/templates/storage-service.yaml @@ -6,6 +6,7 @@ metadata: prometheus.io/path: /minio/v2/metrics/cluster prometheus.io/port: "9000" prometheus.io/scrape: "true" + prometheus.io/direct: "true" {{- with .Values.service.annotations }} {{- toYaml . | nindent 4 }} {{- end }} From 57c89a81521c45309df8cf25521bd0a9af935eba Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 20 Jun 2025 11:45:26 +0800 Subject: [PATCH 06/15] chore(charts): remove _helper.tpl --- charts/storage/templates/_helper.tpl | 29 ------------------- .../templates/storage-secret-creds.yaml | 4 +-- 2 files changed, 2 insertions(+), 31 deletions(-) delete mode 100644 charts/storage/templates/_helper.tpl diff --git a/charts/storage/templates/_helper.tpl b/charts/storage/templates/_helper.tpl deleted file mode 100644 index c35cdc5..0000000 --- a/charts/storage/templates/_helper.tpl +++ /dev/null @@ -1,29 +0,0 @@ -{{- /* keep randAlphaNum values consistent */ -}} -{{- define "storage.accesskey" -}} - {{- if not (index .Release "secrets") -}} - {{- $_ := set .Release "secrets" dict -}} - {{- end -}} - {{- if not (index .Release.secrets "accesskey") -}} - {{- if .Values.accesskey | default "" | ne "" -}} - {{- $_ := set .Release.secrets "accesskey" .Values.accesskey -}} - {{- else -}} - {{- $_ := set .Release.secrets "accesskey" (randAlphaNum 32) -}} - {{- end -}} - {{- end -}} - {{- index .Release.secrets "accesskey" -}} -{{- end -}} - -{{- /* keep randAlphaNum values consistent */ -}} -{{- define "storage.secretkey" -}} - {{- if not (index .Release "secrets") -}} - {{- $_ := set .Release "secrets" dict -}} - {{- end -}} - {{- if not (index .Release.secrets "secretkey") -}} - {{- if .Values.secretkey | default "" | ne "" -}} - {{- $_ := set .Release.secrets "secretkey" .Values.secretkey -}} - {{- else -}} - {{- $_ := set .Release.secrets "secretkey" (randAlphaNum 32) -}} - {{- end -}} - {{- end -}} - {{- index .Release.secrets "secretkey" -}} -{{- end -}} diff --git a/charts/storage/templates/storage-secret-creds.yaml b/charts/storage/templates/storage-secret-creds.yaml index 22f2354..fcfb29f 100644 --- a/charts/storage/templates/storage-secret-creds.yaml +++ b/charts/storage/templates/storage-secret-creds.yaml @@ -6,5 +6,5 @@ metadata: heritage: drycc type: Opaque data: - accesskey: {{ include "common.secrets.lookup" (dict "secret" "storage-creds" "key" "accesskey" "defaultValue" (include "storage.accesskey" .) "context" $) }} - secretkey: {{ include "common.secrets.lookup" (dict "secret" "storage-creds" "key" "secretkey" "defaultValue" (include "storage.secretkey" .) "context" $) }} + accesskey: {{ include "common.secrets.lookup" (dict "secret" "storage-creds" "key" "accesskey" "defaultValue" (default (randAlphaNum 32) .Values.accesskey) "context" $) }} + secretkey: {{ include "common.secrets.lookup" (dict "secret" "storage-creds" "key" "secretkey" "defaultValue" (default (randAlphaNum 32) .Values.secretkey) "context" $) }} From 17d20cc12a873aff63fb809128f9968a19265d65 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 15 Jul 2025 23:37:13 +0800 Subject: [PATCH 07/15] chore(charts): remove cluster domain --- charts/storage/templates/storage-statefulset.yaml | 2 +- charts/storage/values.yaml | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/charts/storage/templates/storage-statefulset.yaml b/charts/storage/templates/storage-statefulset.yaml index 1f1edb8..9982e0e 100644 --- a/charts/storage/templates/storage-statefulset.yaml +++ b/charts/storage/templates/storage-statefulset.yaml @@ -89,7 +89,7 @@ spec: {{- $factor := mul $i $replicaCount }} {{- $endIndex := sub (add $factor $replicaCount) 1 }} {{- $beginIndex := mul $i $replicaCount }} - - {{(printf "http://drycc-storage-{%d...%d}.drycc-storage.%s.svc.%s:9000/data/{0...%d}" $beginIndex $endIndex $.Release.Namespace $.Values.global.clusterDomain (sub $driveCount 1) ) }} + - {{(printf "http://drycc-storage-{%d...%d}.drycc-storage:9000/data/{0...%d}" $beginIndex $endIndex (sub $driveCount 1) ) }} {{- end }} - --address - :9000 diff --git a/charts/storage/values.yaml b/charts/storage/values.yaml index dde6228..0f9b101 100644 --- a/charts/storage/values.yaml +++ b/charts/storage/values.yaml @@ -46,10 +46,3 @@ persistence: enabled: false size: 5Gi storageClass: "" - -global: - # A domain name consists of one or more parts. - # Periods (.) are used to separate these parts. - # Each part must be 1 to 63 characters in length and can contain lowercase letters, digits, and hyphens (-). - # It must start and end with a lowercase letter or digit. - clusterDomain: "cluster.local" From e7673fe31e289ee6e6d3b80d0663a029b2e1350c Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Thu, 17 Jul 2025 15:10:04 +0800 Subject: [PATCH 08/15] chore(storage): set publishNotReadyAddresses --- charts/storage/templates/storage-service.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/storage/templates/storage-service.yaml b/charts/storage/templates/storage-service.yaml index cafa2a5..adf9706 100644 --- a/charts/storage/templates/storage-service.yaml +++ b/charts/storage/templates/storage-service.yaml @@ -14,6 +14,8 @@ metadata: heritage: drycc spec: type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true selector: app: drycc-storage ports: From 4fbb836f686e1e88d038159e61662435741119ee Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Sun, 14 Sep 2025 01:26:27 +0800 Subject: [PATCH 09/15] chore(storage): bump new version --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a244ad8..a36aa21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,8 @@ FROM registry.drycc.cc/drycc/base:bookworm ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/data \ - MC_VERSION="2025.04.03.17.07.56" \ - MINIO_VERSION="2025.04.03.14.56.28" + MC_VERSION="2025.08.13.08.35.41" \ + MINIO_VERSION="2025.09.07.16.13.09" RUN groupadd drycc --gid ${DRYCC_GID} \ From 59ef267fbcc3e66d3dc578e20fdf868ae623738b Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 22 Oct 2025 14:01:07 +0800 Subject: [PATCH 10/15] chore(storage): add CODENAME build-arg --- Dockerfile | 3 ++- Makefile | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a36aa21..2bc2c95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM registry.drycc.cc/drycc/base:bookworm +ARG CODENAME +FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ diff --git a/Makefile b/Makefile index 0b7a16f..ce10cc9 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,9 @@ podman-build: podman build --build-arg CODENAME=${CODENAME} -t ${IMAGE} . podman tag ${IMAGE} ${MUTABLE_IMAGE} +podman-buildx: + podman buildx build --build-arg CODENAME=${CODENAME} --platform ${PLATFORM} -t ${IMAGE} . --push + deploy: build podman-build podman-push .PHONY: all bootstrap build test podman-build deploy From 2b7e2f51c13ae17dc4a668518eb4e51f6f1b0291 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 24 Oct 2025 11:15:33 +0800 Subject: [PATCH 11/15] chore(storage): use rclone replace mc --- Dockerfile | 4 +--- _tests/test.sh | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2bc2c95..810bc54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,14 @@ ARG CODENAME FROM registry.drycc.cc/drycc/base:${CODENAME} -ENV DRYCC_UID=1001 \ +ARG DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/data \ - MC_VERSION="2025.08.13.08.35.41" \ MINIO_VERSION="2025.09.07.16.13.09" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} \ - && install-stack mc $MC_VERSION \ && install-stack minio $MINIO_VERSION \ && rm -rf \ /usr/share/doc \ diff --git a/_tests/test.sh b/_tests/test.sh index e7ee580..e0a3b2a 100755 --- a/_tests/test.sh +++ b/_tests/test.sh @@ -6,7 +6,7 @@ MINIO_ROOT_PASSWORD=f4c4281665bc11ee8e0400163e04a9cd function start-storage { mkdir -p "${BASE_DIR}/data" -podman run --rm -d --name test-storage \ + podman run --rm -d --name test-storage \ -e MINIO_PROMETHEUS_AUTH_TYPE=public \ -e MINIO_ROOT_USER=${MINIO_ROOT_USER} \ -e MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} \ @@ -30,12 +30,20 @@ function main { echo -e "\\033[32m---> Waitting for ${S3_IP}:9000\\033[0m" wait-for-port --host="${S3_IP}" 9000 echo -e "\\033[32m---> S3 service ${S3_IP}:9000 ready...\\033[0m" - # test by minio client - mc --config-dir /tmp/.mc config host add storage "${S3_ENDPOINT}" ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} --lookup path --api s3v4 - mc --config-dir /tmp/.mc mb storage/test - mc --config-dir /tmp/.mc cp "${BASE_DIR}"/test.sh storage/test + # test by rclone client + mkdir -p /tmp/.config/rclone + cat > /tmp/.config/rclone/rclone.conf << EOF +[storage] +type = s3 +provider = Other +endpoint = ${S3_ENDPOINT} +access_key_id = ${MINIO_ROOT_USER} +secret_access_key = ${MINIO_ROOT_PASSWORD} +EOF + rclone --config /tmp/.config/rclone/rclone.conf mkdir storage:test + rclone --config /tmp/.config/rclone/rclone.conf copyto "${BASE_DIR}"/test.sh storage:test/test.sh exit_code=$? - rm -rf /tmp/.mc + rm -rf /tmp/.config/rclone exit $exit_code } From 0e2f17918a7d79c58ec5daad26e9ad0d23de062c Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 7 Nov 2025 10:09:23 +0800 Subject: [PATCH 12/15] feat(storage): change minio to rustfs --- Dockerfile | 8 +- README.md | 2 +- _tests/test.sh | 15 ++- charts/storage/Chart.yaml | 2 +- charts/storage/templates/storage-service.yaml | 7 +- .../templates/storage-statefulset.yaml | 115 +++++++++++++++--- charts/storage/values.yaml | 4 +- rootfs/etc/otelcol/otelcol.yaml | 35 ++++++ 8 files changed, 152 insertions(+), 36 deletions(-) create mode 100644 rootfs/etc/otelcol/otelcol.yaml diff --git a/Dockerfile b/Dockerfile index 810bc54..23a8b48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,12 +4,16 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ARG DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/data \ - MINIO_VERSION="2025.09.07.16.13.09" + RUSTFS_VERSION="1.0.0-alpha.67" \ + OPENTELEMETRY_COLLECTOR_VERSION=0.139.0 +COPY rootfs/etc/otelcol /etc/otelcol RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} \ - && install-stack minio $MINIO_VERSION \ + && install-packages dnsutils \ + && install-stack rustfs $RUSTFS_VERSION \ + && install-stack opentelemetry-collector $OPENTELEMETRY_COLLECTOR_VERSION \ && rm -rf \ /usr/share/doc \ /usr/share/man \ diff --git a/README.md b/README.md index 75e31ea..9b5e528 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ We welcome your input! If you have feedback, please submit an [issue][issues]. I # About -The Drycc storage component provides an [S3 API][s3-api] compatible object storage server, based on [MINIO](https://github.com/minio/minio), that can be run on Kubernetes. It's intended for use within the [Drycc v2 platform][drycc-docs] as an object storage server, but it's flexible enough to be run as a standalone pod on any Kubernetes cluster. +The Drycc storage component provides an [S3 API][s3-api] compatible object storage server, based on [RUSTFS](https://github.com/rustfs/rustfs), that can be run on Kubernetes. It's intended for use within the [Drycc v2 platform][drycc-docs] as an object storage server, but it's flexible enough to be run as a standalone pod on any Kubernetes cluster. Note that in the default [Helm chart for the Drycc platform](https://github.com/drycc/charts/tree/main/drycc-dev), this component is used as a storage location for the following components: diff --git a/_tests/test.sh b/_tests/test.sh index e0a3b2a..5fd09b8 100755 --- a/_tests/test.sh +++ b/_tests/test.sh @@ -1,17 +1,16 @@ #!/usr/bin/env bash BASE_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") -MINIO_ROOT_USER=f4c4281665bc11ee8e0400163e04a9cd -MINIO_ROOT_PASSWORD=f4c4281665bc11ee8e0400163e04a9cd +RUSTFS_ACCESS_KEY=f4c4281665bc11ee8e0400163e04a9cd +RUSTFS_SECRET_KEY=f4c4281665bc11ee8e0400163e04a9cd function start-storage { mkdir -p "${BASE_DIR}/data" podman run --rm -d --name test-storage \ - -e MINIO_PROMETHEUS_AUTH_TYPE=public \ - -e MINIO_ROOT_USER=${MINIO_ROOT_USER} \ - -e MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} \ + -e RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY} \ + -e RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY} \ registry.drycc.cc/drycc/storage:canary \ - minio server /tmp --address :9000 --console-address :9001 + rustfs /tmp --address :9000 --console-address :9001 } # shellcheck disable=SC2317 @@ -37,8 +36,8 @@ function main { type = s3 provider = Other endpoint = ${S3_ENDPOINT} -access_key_id = ${MINIO_ROOT_USER} -secret_access_key = ${MINIO_ROOT_PASSWORD} +access_key_id = ${RUSTFS_ACCESS_KEY} +secret_access_key = ${RUSTFS_SECRET_KEY} EOF rclone --config /tmp/.config/rclone/rclone.conf mkdir storage:test rclone --config /tmp/.config/rclone/rclone.conf copyto "${BASE_DIR}"/test.sh storage:test/test.sh diff --git a/charts/storage/Chart.yaml b/charts/storage/Chart.yaml index 7ebd898..185f568 100644 --- a/charts/storage/Chart.yaml +++ b/charts/storage/Chart.yaml @@ -6,7 +6,7 @@ dependencies: - name: common repository: oci://registry.drycc.cc/charts version: ~1.1.2 -description: Minio Object Storage in Kubernetes, used by Drycc Workflow. +description: Object Storage in Kubernetes, used by Drycc Workflow. maintainers: - name: Drycc Team email: engineering@drycc.com diff --git a/charts/storage/templates/storage-service.yaml b/charts/storage/templates/storage-service.yaml index adf9706..959e1bf 100644 --- a/charts/storage/templates/storage-service.yaml +++ b/charts/storage/templates/storage-service.yaml @@ -3,10 +3,9 @@ kind: Service metadata: name: drycc-storage annotations: - prometheus.io/path: /minio/v2/metrics/cluster - prometheus.io/port: "9000" + prometheus.io/path: /metrics + prometheus.io/port: "9200" prometheus.io/scrape: "true" - prometheus.io/direct: "true" {{- with .Values.service.annotations }} {{- toYaml . | nindent 4 }} {{- end }} @@ -23,3 +22,5 @@ spec: port: 9000 - name: console port: 9001 + - name: metrics + port: 9200 diff --git a/charts/storage/templates/storage-statefulset.yaml b/charts/storage/templates/storage-statefulset.yaml index 9982e0e..3346f81 100644 --- a/charts/storage/templates/storage-statefulset.yaml +++ b/charts/storage/templates/storage-statefulset.yaml @@ -1,6 +1,19 @@ {{- $zoneCount := int .Values.zones }} {{- $driveCount := int .Values.drivesPerNode }} {{- $replicaCount := int .Values.replicas }} +{{- $volumesList := list }} +{{- $hostsList := list }} +{{- range $i := until $zoneCount }} +{{- $factor := mul $i $replicaCount }} +{{- $endIndex := sub (add $factor $replicaCount) 1 }} +{{- $beginIndex := mul $i $replicaCount }} +{{- $volumes := (printf "http://drycc-storage-{%d...%d}.drycc-storage:9000/data/{0...%d}" $beginIndex $endIndex (sub $driveCount 1) ) }} +{{- $volumesList = append $volumesList $volumes }} +{{- range $j := until $replicaCount }} +{{- $nodeIndex := add $factor $j }} +{{- $hostsList = append $hostsList (printf "drycc-storage-%d.drycc-storage" $nodeIndex) }} +{{- end }} +{{- end }} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -20,6 +33,10 @@ spec: app: drycc-storage template: metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "8888" + prometheus.io/scrape: "true" labels: app: drycc-storage spec: @@ -28,19 +45,42 @@ spec: podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }} nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values ) | nindent 10 }} serviceAccount: drycc-storage + initContainers: + - name: dns-check + image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/storage:{{.Values.imageTag}} + imagePullPolicy: {{.Values.imagePullPolicy}} + command: + - /usr/bin/env + - bash + - -ec + - | + echo "Checking DNS resolution for storage hosts..." + # Check all hosts in hostsList + hosts=({{- range $host := $hostsList }} "{{$host}}"{{- end }}) + for host in "${hosts[@]}"; do + echo "Checking DNS resolution for $host..." + while true; do + if nslookup "$host" > /dev/null 2>&1; then + echo "DNS resolution for $host: OK" + break + else + echo "DNS resolution for $host failed, retrying in 5 seconds..." + sleep 5 + fi + done + done + echo "All DNS checks passed, ready to start storage containers." containers: - name: drycc-storage image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/storage:{{.Values.imageTag}} imagePullPolicy: {{.Values.imagePullPolicy}} env: - - name: MINIO_PROMETHEUS_AUTH_TYPE - value: "public" - - name: "MINIO_ROOT_USER" + - name: "RUSTFS_ACCESS_KEY" valueFrom: secretKeyRef: name: storage-creds key: accesskey - - name: "MINIO_ROOT_PASSWORD" + - name: "RUSTFS_SECRET_KEY" valueFrom: secretKeyRef: name: storage-creds @@ -58,7 +98,7 @@ spec: {{- end }} livenessProbe: httpGet: - path: /minio/health/live + path: /health port: 9000 initialDelaySeconds: 5 periodSeconds: 5 @@ -76,30 +116,67 @@ spec: startupProbe: tcpSocket: port: 9000 - initialDelaySeconds: 0 + initialDelaySeconds: 30 periodSeconds: 10 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 60 args: - - minio - - server - {{- $clusters := list }} - {{- range $i := until $zoneCount }} - {{- $factor := mul $i $replicaCount }} - {{- $endIndex := sub (add $factor $replicaCount) 1 }} - {{- $beginIndex := mul $i $replicaCount }} - - {{(printf "http://drycc-storage-{%d...%d}.drycc-storage:9000/data/{0...%d}" $beginIndex $endIndex (sub $driveCount 1) ) }} - {{- end }} - - --address - - :9000 - - --console-address - - :9001 + - rustfs + - {{ join " " $volumesList }} + - --address=:9000 + - --console-address=:9001 + - --obs-endpoint=http://localhost:4317 volumeMounts: {{- range $diskId := until $driveCount }} - name: storage-data-{{$diskId}} mountPath: /data/{{$diskId}} {{- end }} + - name: drycc-storage-otelcol + image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/storage:{{.Values.imageTag}} + imagePullPolicy: {{.Values.imagePullPolicy}} + ports: + - name: receiver + containerPort: 4317 + protocol: TCP + - name: otelcol + containerPort: 9200 + protocol: TCP + - name: metrics + containerPort: 8888 + protocol: TCP + {{- with index .Values "resources" }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + livenessProbe: + httpGet: + path: /metrics + port: 9200 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + tcpSocket: + port: 9200 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + startupProbe: + tcpSocket: + port: 9200 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + args: + - otelcol + - --config=/etc/otelcol/otelcol.yaml securityContext: fsGroup: 1001 runAsGroup: 1001 diff --git a/charts/storage/values.yaml b/charts/storage/values.yaml index 0f9b101..1a9e68f 100644 --- a/charts/storage/values.yaml +++ b/charts/storage/values.yaml @@ -11,7 +11,7 @@ resources: {} # cpu: 100m # memory: 30Mi -# The following parameters are configured only when using an on-cluster Minio instance +# The following parameters are configured only when using an on-cluster object storage instance nodeAffinityPreset: key: "drycc.cc/node" type: "soft" @@ -28,7 +28,7 @@ podAntiAffinityPreset: # Number of replicas per zone replicas: 4 -# Minio zone count +# Object storage zone count zones: 1 # Device count of per replica drivesPerNode: 1 diff --git a/rootfs/etc/otelcol/otelcol.yaml b/rootfs/etc/otelcol/otelcol.yaml new file mode 100644 index 0000000..bdc0575 --- /dev/null +++ b/rootfs/etc/otelcol/otelcol.yaml @@ -0,0 +1,35 @@ +# Simplified version: Focus on OTLP metrics collection and Prometheus export +receivers: + otlp: # OTLP protocol receiver + protocols: + grpc: # OTLP gRPC receiver for rustfs metrics data + endpoint: 0.0.0.0:4317 # gRPC endpoint for receiving metrics +processors: + batch: # Batch processor to improve throughput + timeout: 5s # Maximum time to wait before sending a batch + send_batch_size: 1000 # Maximum batch size + memory_limiter: # Memory limiter to prevent OOM + check_interval: 1s # Interval to check memory usage + limit_mib: 256 # Reduce memory limit to 256MB +exporters: + prometheus: # Prometheus exporter for Grafana scraping + endpoint: "0.0.0.0:9200" # Endpoint for Grafana to scrape metrics + namespace: "rustfs" # Metrics prefix (will prefix all metrics with "rustfs_") + send_timestamps: true # Include timestamps with metrics +service: + pipelines: + metrics: # Keep only metrics pipeline + receivers: [otlp] # Receive OTLP metrics from rustfs + processors: [memory_limiter, batch] # Apply memory limiting and batching + exporters: [prometheus] # Export to Prometheus for Grafana + telemetry: + logs: + level: "warn" # Reduce log level to minimize output + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 # Host for internal metrics endpoint + port: 8888 # Port for internal metrics endpoint + without_units: true # Export metrics without units for compatibility From 280ab576136ca79c90390ba1a3090adf59f80878 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Thu, 20 Nov 2025 09:47:27 +0800 Subject: [PATCH 13/15] chore(rustfs): bump version to 1.0.0-alpha.69 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 23a8b48..2c301e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ARG DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/data \ - RUSTFS_VERSION="1.0.0-alpha.67" \ + RUSTFS_VERSION="1.0.0-alpha.69" \ OPENTELEMETRY_COLLECTOR_VERSION=0.139.0 COPY rootfs/etc/otelcol /etc/otelcol From cc11e90799236a13388a3e801866b8e12d0ff30c Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Thu, 4 Dec 2025 17:46:15 +0800 Subject: [PATCH 14/15] chore(rustfs): bump version to 1.0.0-alpha.71 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2c301e0..ccdb0b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ARG DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/data \ - RUSTFS_VERSION="1.0.0-alpha.69" \ + RUSTFS_VERSION="1.0.0-alpha.71" \ OPENTELEMETRY_COLLECTOR_VERSION=0.139.0 COPY rootfs/etc/otelcol /etc/otelcol From 71050f88965d012ea9e6e81849cae125fda95e60 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 12 Dec 2025 08:54:44 +0800 Subject: [PATCH 15/15] chore(storage): bump rustfs version to 1.0.0-alpha.73 --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ccdb0b8..7425306 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,8 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ARG DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/data \ - RUSTFS_VERSION="1.0.0-alpha.71" \ - OPENTELEMETRY_COLLECTOR_VERSION=0.139.0 + RUSTFS_VERSION="1.0.0-alpha.73" \ + OPENTELEMETRY_COLLECTOR_VERSION=0.141.0 COPY rootfs/etc/otelcol /etc/otelcol