feat(storage): change minio to rustfs

duanhongyi · duanhongyi · commit 0e2f17918a7d · 2025-11-07T16:19:20.000+08:00
diff --git a/Dockerfile b/Dockerfile
@@ -4,12 +4,16 @@ FROM registry.drycc.cc/drycc/base:${CODENAME}
 ARG DRYCC_UID=1001 \
   DRYCC_GID=1001 \
   DRYCC_HOME_DIR=/data \
-  MINIO_VERSION="2025.09.07.16.13.09"
+  RUSTFS_VERSION="1.0.0-alpha.67" \
+  OPENTELEMETRY_COLLECTOR_VERSION=0.139.0
 
+COPY rootfs/etc/otelcol /etc/otelcol
 
 RUN groupadd drycc --gid ${DRYCC_GID} \
   && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} \
-  && install-stack minio $MINIO_VERSION \
+  && install-packages dnsutils \
+  && install-stack rustfs $RUSTFS_VERSION \
+  && install-stack opentelemetry-collector $OPENTELEMETRY_COLLECTOR_VERSION \
   && rm -rf \
       /usr/share/doc \
       /usr/share/man \
diff --git a/README.md b/README.md
@@ -10,7 +10,7 @@ We welcome your input! If you have feedback, please submit an [issue][issues]. I
 
 # About
 
-The Drycc storage component provides an [S3 API][s3-api] compatible object storage server, based on [MINIO](https://github.com/minio/minio), that can be run on Kubernetes. It's intended for use within the [Drycc v2 platform][drycc-docs] as an object storage server, but it's flexible enough to be run as a standalone pod on any Kubernetes cluster.
+The Drycc storage component provides an [S3 API][s3-api] compatible object storage server, based on [RUSTFS](https://github.com/rustfs/rustfs), that can be run on Kubernetes. It's intended for use within the [Drycc v2 platform][drycc-docs] as an object storage server, but it's flexible enough to be run as a standalone pod on any Kubernetes cluster.
 
 Note that in the default [Helm chart for the Drycc platform](https://github.com/drycc/charts/tree/main/drycc-dev), this component is used as a storage location for the following components:
 
diff --git a/_tests/test.sh b/_tests/test.sh
@@ -1,17 +1,16 @@
 #!/usr/bin/env bash
 
 BASE_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
-MINIO_ROOT_USER=f4c4281665bc11ee8e0400163e04a9cd
-MINIO_ROOT_PASSWORD=f4c4281665bc11ee8e0400163e04a9cd
+RUSTFS_ACCESS_KEY=f4c4281665bc11ee8e0400163e04a9cd
+RUSTFS_SECRET_KEY=f4c4281665bc11ee8e0400163e04a9cd
 
 function start-storage {
   mkdir -p "${BASE_DIR}/data"
   podman run --rm -d --name test-storage \
-    -e MINIO_PROMETHEUS_AUTH_TYPE=public \
-    -e MINIO_ROOT_USER=${MINIO_ROOT_USER} \
-    -e MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} \
+    -e RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY} \
+    -e RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY} \
     registry.drycc.cc/drycc/storage:canary \
-    minio server /tmp --address :9000 --console-address :9001
+    rustfs /tmp --address :9000 --console-address :9001
 }
 
 # shellcheck disable=SC2317
@@ -37,8 +36,8 @@ function main {
 type = s3
 provider = Other
 endpoint = ${S3_ENDPOINT}
-access_key_id = ${MINIO_ROOT_USER}
-secret_access_key = ${MINIO_ROOT_PASSWORD}
+access_key_id = ${RUSTFS_ACCESS_KEY}
+secret_access_key = ${RUSTFS_SECRET_KEY}
 EOF
   rclone --config /tmp/.config/rclone/rclone.conf mkdir storage:test
   rclone --config /tmp/.config/rclone/rclone.conf copyto "${BASE_DIR}"/test.sh storage:test/test.sh
diff --git a/charts/storage/Chart.yaml b/charts/storage/Chart.yaml
@@ -6,7 +6,7 @@ dependencies:
 - name: common
   repository: oci://registry.drycc.cc/charts
   version: ~1.1.2
-description: Minio Object Storage in Kubernetes, used by Drycc Workflow.
+description: Object Storage in Kubernetes, used by Drycc Workflow.
 maintainers:
 - name: Drycc Team
   email: engineering@drycc.com
diff --git a/charts/storage/templates/storage-service.yaml b/charts/storage/templates/storage-service.yaml
@@ -3,10 +3,9 @@ kind: Service
 metadata:
   name: drycc-storage
   annotations:
-    prometheus.io/path: /minio/v2/metrics/cluster
-    prometheus.io/port: "9000"
+    prometheus.io/path: /metrics
+    prometheus.io/port: "9200"
     prometheus.io/scrape: "true"
-    prometheus.io/direct: "true"
     {{- with .Values.service.annotations }}
     {{- toYaml . | nindent 4 }}
     {{- end }}
@@ -23,3 +22,5 @@ spec:
     port: 9000
   - name: console
     port: 9001
+  - name: metrics
+    port: 9200
diff --git a/charts/storage/templates/storage-statefulset.yaml b/charts/storage/templates/storage-statefulset.yaml
@@ -1,6 +1,19 @@
 {{- $zoneCount := int .Values.zones }}
 {{- $driveCount := int .Values.drivesPerNode }}
 {{- $replicaCount := int .Values.replicas }}
+{{- $volumesList := list }}
+{{- $hostsList := list }}
+{{- range $i := until $zoneCount }}
+{{- $factor := mul $i $replicaCount }}
+{{- $endIndex := sub (add $factor $replicaCount) 1 }}
+{{- $beginIndex := mul $i $replicaCount }}
+{{- $volumes := (printf "http://drycc-storage-{%d...%d}.drycc-storage:9000/data/{0...%d}" $beginIndex $endIndex (sub $driveCount 1) ) }}
+{{- $volumesList = append $volumesList $volumes }}
+{{- range $j := until $replicaCount }}
+{{- $nodeIndex := add $factor $j }}
+{{- $hostsList = append $hostsList (printf "drycc-storage-%d.drycc-storage" $nodeIndex) }}
+{{- end }}
+{{- end }}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -20,6 +33,10 @@ spec:
       app: drycc-storage
   template:
     metadata:
+      annotations:
+        prometheus.io/path: /metrics
+        prometheus.io/port: "8888"
+        prometheus.io/scrape: "true"
       labels:
         app: drycc-storage
     spec:
@@ -28,19 +45,42 @@ spec:
         podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
         nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values ) | nindent 10 }}
       serviceAccount: drycc-storage
+      initContainers:
+      - name: dns-check
+        image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/storage:{{.Values.imageTag}}
+        imagePullPolicy: {{.Values.imagePullPolicy}}
+        command:
+        - /usr/bin/env
+        - bash
+        - -ec
+        - |
+          echo "Checking DNS resolution for storage hosts..."
+          # Check all hosts in hostsList
+          hosts=({{- range $host := $hostsList }} "{{$host}}"{{- end }})
+          for host in "${hosts[@]}"; do
+            echo "Checking DNS resolution for $host..."
+            while true; do
+              if nslookup "$host" > /dev/null 2>&1; then
+                echo "DNS resolution for $host: OK"
+                break
+              else
+                echo "DNS resolution for $host failed, retrying in 5 seconds..."
+                sleep 5
+              fi
+            done
+          done
+          echo "All DNS checks passed, ready to start storage containers."
       containers:
       - name: drycc-storage
         image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/storage:{{.Values.imageTag}}
         imagePullPolicy: {{.Values.imagePullPolicy}}
         env:
-        - name: MINIO_PROMETHEUS_AUTH_TYPE
-          value: "public"
-        - name: "MINIO_ROOT_USER"
+        - name: "RUSTFS_ACCESS_KEY"
           valueFrom:
             secretKeyRef:
               name: storage-creds
               key: accesskey
-        - name: "MINIO_ROOT_PASSWORD"
+        - name: "RUSTFS_SECRET_KEY"
           valueFrom:
             secretKeyRef:
               name: storage-creds
@@ -58,7 +98,7 @@ spec:
         {{- end }}
         livenessProbe:
           httpGet:
-            path: /minio/health/live
+            path: /health
             port: 9000
           initialDelaySeconds: 5
           periodSeconds: 5
@@ -76,30 +116,67 @@ spec:
         startupProbe:
           tcpSocket:
             port: 9000
-          initialDelaySeconds: 0
+          initialDelaySeconds: 30
           periodSeconds: 10
           timeoutSeconds: 5
           successThreshold: 1
           failureThreshold: 60
         args:
-        - minio
-        - server
-        {{- $clusters := list }}
-        {{- range $i := until $zoneCount }}
-        {{- $factor := mul $i $replicaCount }}
-        {{- $endIndex := sub (add $factor $replicaCount) 1 }}
-        {{- $beginIndex := mul $i $replicaCount }}
-        - {{(printf "http://drycc-storage-{%d...%d}.drycc-storage:9000/data/{0...%d}" $beginIndex $endIndex (sub $driveCount 1) ) }}
-        {{- end }}
-        - --address
-        - :9000
-        - --console-address
-        - :9001
+        - rustfs
+        - {{ join " " $volumesList }}
+        - --address=:9000
+        - --console-address=:9001
+        - --obs-endpoint=http://localhost:4317
         volumeMounts:
         {{- range $diskId := until $driveCount }}
         - name: storage-data-{{$diskId}}
           mountPath: /data/{{$diskId}}
         {{- end }}
+      - name: drycc-storage-otelcol
+        image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/storage:{{.Values.imageTag}}
+        imagePullPolicy: {{.Values.imagePullPolicy}}
+        ports:
+        - name: receiver
+          containerPort: 4317
+          protocol: TCP
+        - name: otelcol
+          containerPort: 9200
+          protocol: TCP
+        - name: metrics
+          containerPort: 8888
+          protocol: TCP
+        {{- with index .Values "resources" }}
+        resources:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        livenessProbe:
+          httpGet:
+            path: /metrics
+            port: 9200
+          initialDelaySeconds: 5
+          periodSeconds: 5
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 5
+        readinessProbe:
+          tcpSocket:
+            port: 9200
+          initialDelaySeconds: 5
+          periodSeconds: 5
+          timeoutSeconds: 1
+          successThreshold: 1
+          failureThreshold: 5
+        startupProbe:
+          tcpSocket:
+            port: 9200
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 60
+        args:
+        - otelcol
+        - --config=/etc/otelcol/otelcol.yaml
       securityContext:
         fsGroup: 1001
         runAsGroup: 1001
diff --git a/charts/storage/values.yaml b/charts/storage/values.yaml
@@ -11,7 +11,7 @@ resources: {}
   #   cpu: 100m
   #   memory: 30Mi
 
-# The following parameters are configured only when using an on-cluster Minio instance
+# The following parameters are configured only when using an on-cluster object storage instance
 nodeAffinityPreset:
   key: "drycc.cc/node"
   type: "soft"
@@ -28,7 +28,7 @@ podAntiAffinityPreset:
 
 # Number of replicas per zone
 replicas: 4
-# Minio zone count
+# Object storage zone count
 zones: 1
 # Device count of per replica
 drivesPerNode: 1
diff --git a/rootfs/etc/otelcol/otelcol.yaml b/rootfs/etc/otelcol/otelcol.yaml
@@ -0,0 +1,35 @@
+# Simplified version: Focus on OTLP metrics collection and Prometheus export
+receivers:
+  otlp:                                      # OTLP protocol receiver
+    protocols:
+      grpc:                                  # OTLP gRPC receiver for rustfs metrics data
+        endpoint: 0.0.0.0:4317               # gRPC endpoint for receiving metrics
+processors:
+  batch:                                     # Batch processor to improve throughput
+    timeout: 5s                              # Maximum time to wait before sending a batch
+    send_batch_size: 1000                    # Maximum batch size
+  memory_limiter:                            # Memory limiter to prevent OOM
+    check_interval: 1s                       # Interval to check memory usage
+    limit_mib: 256                           # Reduce memory limit to 256MB
+exporters:
+  prometheus:                                # Prometheus exporter for Grafana scraping
+    endpoint: "0.0.0.0:9200"                 # Endpoint for Grafana to scrape metrics
+    namespace: "rustfs"                      # Metrics prefix (will prefix all metrics with "rustfs_")
+    send_timestamps: true                    # Include timestamps with metrics
+service:
+  pipelines:
+    metrics:                                 # Keep only metrics pipeline
+      receivers: [otlp]                      # Receive OTLP metrics from rustfs
+      processors: [memory_limiter, batch]    # Apply memory limiting and batching
+      exporters: [prometheus]                # Export to Prometheus for Grafana
+  telemetry:
+    logs:
+      level: "warn"                          # Reduce log level to minimize output
+    metrics:
+      readers:
+      - pull:
+          exporter:
+            prometheus:
+              host: 0.0.0.0                  # Host for internal metrics endpoint
+              port: 8888                     # Port for internal metrics endpoint
+              without_units: true            # Export metrics without units for compatibility
