diff --git a/.gitignore b/.gitignore index 1b73338..e5361b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ -rootfs/opt/ contrib/ci/tmp/ .vscode/ +rootfs/opt/registry/sbin \ No newline at end of file diff --git a/.woodpecker/build-linux.yml b/.woodpecker/build-linux.yml index 8cdd984..ef4ecfd 100644 --- a/.woodpecker/build-linux.yml +++ b/.woodpecker/build-linux.yml @@ -3,28 +3,33 @@ matrix: - linux/amd64 - linux/arm64 -platform: ${platform} - labels: type: exec + platform: ${platform} -pipeline: +steps: - name: publish-linux image: bash commands: - - export VERSION=$([ -z $CI_COMMIT_TAG ] && echo latest || echo $CI_COMMIT_TAG)-$(sed 's#/#-#g' <<< $CI_SYSTEM_ARCH) + - export VERSION=$([ -z $CI_COMMIT_TAG ] && echo latest || echo $CI_COMMIT_TAG)-$(sed 's#/#-#g' <<< $CI_SYSTEM_PLATFORM) - echo $CONTAINER_PASSWORD | podman login $DRYCC_REGISTRY --username $CONTAINER_USERNAME --password-stdin > /dev/null 2>&1 - make podman-build podman-immutable-push - secrets: - - codename - - dev_registry - - drycc_registry - - container_username - - container_password + environment: + CODENAME: + from_secret: codename + DEV_REGISTRY: + from_secret: dev_registry + DRYCC_REGISTRY: + from_secret: drycc_registry + CONTAINER_USERNAME: + from_secret: container_username + CONTAINER_PASSWORD: + from_secret: container_password when: event: - push - tag + - cron depends_on: - test-linux \ No newline at end of file diff --git a/.woodpecker/chart.yaml b/.woodpecker/chart.yaml index 3204cf5..cfcd3ec 100644 --- a/.woodpecker/chart.yaml +++ b/.woodpecker/chart.yaml @@ -1,30 +1,33 @@ -platform: linux/amd64 - labels: type: exec + platform: linux/amd64 -pipeline: +steps: - name: generate-chart - type: local image: bash commands: - export VERSION=$(sed 's#v##' <<< $CI_COMMIT_TAG) - export IMAGE_TAG=$([ ! -z $CI_COMMIT_TAG ] && echo \"$VERSION\" || echo \"canary\") - - export APP_VERSION=$([ ! -z $CI_COMMIT_TAG ] && echo $VERSION || echo 1.0.0) + - export APP_VERSION=$([ -z $CI_COMMIT_TAG ] && echo $CI_COMMIT_SHA || echo $VERSION) - export CHART_VERSION=$([ -z $CI_COMMIT_TAG ] && echo 1.0.0 || echo $VERSION) - sed -i "s/imageTag:\ \"canary\"/imageTag:\ $IMAGE_TAG/g" charts/$${CI_REPO_NAME}/values.yaml - helm package -u charts/$${CI_REPO_NAME} --version $CHART_VERSION --app-version $APP_VERSION - echo $CONTAINER_PASSWORD | helm registry login $DRYCC_REGISTRY -u $CONTAINER_USERNAME --password-stdin - helm push $${CI_REPO_NAME}-$CHART_VERSION.tgz oci://$DRYCC_REGISTRY/$([ -z $CI_COMMIT_TAG ] && echo charts-testing || echo charts) - secrets: - - dev_registry - - drycc_registry - - container_username - - container_password + environment: + DEV_REGISTRY: + from_secret: dev_registry + DRYCC_REGISTRY: + from_secret: drycc_registry + CONTAINER_USERNAME: + from_secret: container_username + CONTAINER_PASSWORD: + from_secret: container_password when: event: - push - tag + - cron depends_on: -- manifest \ No newline at end of file +- manifest diff --git a/.woodpecker/manifest.yml b/.woodpecker/manifest.yml index 2d5c4e5..19fc70c 100644 --- a/.woodpecker/manifest.yml +++ b/.woodpecker/manifest.yml @@ -1,20 +1,21 @@ -platform: linux/amd64 - labels: type: exec + platform: linux/amd64 -pipeline: +steps: - name: generate-manifest image: bash commands: - sed -i "s/{{project}}/$${CI_REPO_NAME}/g" .woodpecker/manifest.tmpl - sed -i "s/registry.drycc.cc/$${DRYCC_REGISTRY}/g" .woodpecker/manifest.tmpl - secrets: - - drycc_registry + environment: + DRYCC_REGISTRY: + from_secret: drycc_registry when: event: - tag - push + - cron - name: publish-manifest image: bash @@ -27,13 +28,16 @@ pipeline: -v $(pwd):$(pwd) -w $(pwd) docker.io/plugins/manifest - secrets: - - container_username - - container_password + environment: + CONTAINER_USERNAME: + from_secret: container_username + CONTAINER_PASSWORD: + from_secret: container_password when: event: - tag - push + - cron depends_on: - build-linux diff --git a/.woodpecker/test-linux.yml b/.woodpecker/test-linux.yml index 313179e..194832f 100644 --- a/.woodpecker/test-linux.yml +++ b/.woodpecker/test-linux.yml @@ -3,20 +3,22 @@ matrix: - linux/amd64 - linux/arm64 -platform: ${platform} - labels: type: exec + platform: ${platform} -pipeline: +steps: - name: test-linux image: bash commands: - make test - secrets: - - codename - - dev_registry + environment: + CODENAME: + from_secret: codename + DEV_REGISTRY: + from_secret: dev_registry when: event: - push - tag + - cron diff --git a/Dockerfile b/Dockerfile index cc66961..e9e37c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,29 +5,27 @@ ARG LDFLAGS ADD . /workspace RUN export GO111MODULE=on \ && cd /workspace \ - && CGO_ENABLED=0 init-stack go build -ldflags "${LDFLAGS}" -o /bin/boot main.go \ - && upx -9 --brute /bin/boot + && CGO_ENABLED=0 init-stack go build -ldflags "${LDFLAGS}" -o /bin/start-registry main.go \ + && upx -9 --brute /bin/start-registry FROM registry.drycc.cc/drycc/base:${CODENAME} -ENV DRYCC_UID=1001 \ +ARG DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/var/lib/registry \ - JQ_VERSION="1.7" \ - MC_VERSION="2023.09.20.15.22.31" \ - REGISTRY_VERSION="2.8.3" - -COPY rootfs/bin/ /bin/ -COPY --from=build /bin/boot /bin/boot + JQ_VERSION="1.7.1" \ + NGINX_VERSION="1.29.1" \ + RCLONE_VERSION="1.71.1" \ + REGISTRY_VERSION="3.0.0" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} \ && install-packages apache2-utils \ && install-stack jq $JQ_VERSION \ - && install-stack mc $MC_VERSION \ + && install-stack nginx ${NGINX_VERSION} \ + && install-stack rclone $RCLONE_VERSION \ && install-stack registry $REGISTRY_VERSION \ - && chmod +x /bin/init_registry \ && rm -rf \ /usr/share/doc \ /usr/share/man \ @@ -42,12 +40,16 @@ RUN groupadd drycc --gid ${DRYCC_GID} \ /usr/lib/`echo $(uname -m)`-linux-gnu/gconv/IBM* \ /usr/lib/`echo $(uname -m)`-linux-gnu/gconv/EBC* \ && mkdir -p /usr/share/man/man{1..8} \ - && chown -R ${DRYCC_UID}:${DRYCC_GID} ${DRYCC_HOME_DIR} + && chown -R ${DRYCC_UID}:${DRYCC_GID} /opt/drycc +COPY --from=build /bin/start-registry /bin/start-registry +COPY --chown=${DRYCC_UID}:${DRYCC_GID} rootfs/bin/ /bin/ +COPY --chown=${DRYCC_UID}:${DRYCC_GID} rootfs/opt/drycc/nginx /opt/drycc/nginx COPY --chown=${DRYCC_UID}:${DRYCC_GID} rootfs/config-example.yml /opt/drycc/registry/etc/config.yml -ENV DRYCC_REGISTRY_CONFIG /opt/drycc/registry/etc/config.yml + +ENV OTEL_TRACES_EXPORTER=none \ + DRYCC_REGISTRY_CONFIG=/opt/drycc/registry/etc/config.yml USER ${DRYCC_UID} VOLUME ["${DRYCC_HOME_DIR}"] -CMD ["/bin/boot"] EXPOSE 5000 diff --git a/README.md b/README.md index 5e9ea3d..ae7eea3 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,15 @@ We welcome your input! If you have feedback, please submit an [issue][issues]. I # About -The registry is a [Container registry](https://github.com/distribution/distribution) component for use in Kubernetes. While it's intended for use inside of the Drycc open source [PaaS](https://en.wikipedia.org/wiki/Platform_as_a_service), it's flexible enough to be used as a standalone pod on any Kubernetes cluster. +Registry consists of two components, namely the proxy component and the registry component. + +## Proxy + +The proxy component is a proxy deployed on every Kubernetes worker node, proxying all requests to the Drycc Workflow [registry][registry]. This allows the worker nodes daemons to communicate to the registry over localhost, bypassing the need for adding the `--insecure-registry` flag to the daemons. + +## Registry + +The registry component is a [Container registry](https://github.com/distribution/distribution) component for use in Kubernetes. While it's intended for use inside of the Drycc open source [PaaS](https://en.wikipedia.org/wiki/Platform_as_a_service), it's flexible enough to be used as a standalone pod on any Kubernetes cluster. If you decide to use this component standalone, you can host your own Container registry in your own Kubernetes cluster. diff --git a/charts/registry/templates/_helper.tpl b/charts/registry/templates/_helper.tpl index a4e066b..cd87b72 100644 --- a/charts/registry/templates/_helper.tpl +++ b/charts/registry/templates/_helper.tpl @@ -24,21 +24,37 @@ env: secretKeyRef: name: registry-secret key: password -- name: "DRYCC_STORAGE_LOOKUP" - valueFrom: - secretKeyRef: - name: storage-creds - key: lookup +{{- if (.Values.storageEndpoint) }} - name: "DRYCC_STORAGE_BUCKET" valueFrom: secretKeyRef: - name: storage-creds - key: registry-bucket + name: registry-secret + key: storage-bucket - name: "DRYCC_STORAGE_ENDPOINT" valueFrom: secretKeyRef: - name: storage-creds - key: endpoint + name: registry-secret + key: storage-endpoint +- name: "DRYCC_STORAGE_ACCESSKEY" + valueFrom: + secretKeyRef: + name: registry-secret + key: storage-accesskey +- name: "DRYCC_STORAGE_SECRETKEY" + valueFrom: + secretKeyRef: + name: registry-secret + key: storage-secretkey +- name: "DRYCC_STORAGE_PATH_STYLE" + valueFrom: + secretKeyRef: + name: registry-secret + key: storage-path-style +{{- else if .Values.storage.enabled }} +- name: "DRYCC_STORAGE_BUCKET" + value: "registry" +- name: "DRYCC_STORAGE_ENDPOINT" + value: http://drycc-storage:9000 - name: "DRYCC_STORAGE_ACCESSKEY" valueFrom: secretKeyRef: @@ -49,24 +65,7 @@ env: secretKeyRef: name: storage-creds key: secretkey +- name: "DRYCC_STORAGE_PATH_STYLE" + value: "true" {{- end }} - -{{/* Generate registry deployment limits */}} -{{- define "registry.limits" -}} -{{- if or (.Values.limitsCpu) (.Values.limitsMemory)}} -resources: - limits: - {{- if (.Values.limitsCpu) }} - cpu: {{.Values.limitsCpu}} - {{- end }} - {{- if (.Values.limitsMemory) }} - memory: {{.Values.limitsMemory}} - {{- end }} - {{- if (.Values.limitsHugepages2Mi) }} - hugepages-2Mi: {{.Values.limitsHugepages2Mi}} - {{- end }} - {{- if (.Values.limitsHugepages1Gi) }} - hugepages-1Gi: {{.Values.limitsHugepages1Gi}} - {{- end }} {{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/registry/templates/registry-cronjob-daily.yaml b/charts/registry/templates/registry-cronjob-daily.yaml index 60f89b7..442841b 100644 --- a/charts/registry/templates/registry-cronjob-daily.yaml +++ b/charts/registry/templates/registry-cronjob-daily.yaml @@ -1,4 +1,3 @@ -{{- if eq .Values.global.registryLocation "on-cluster" }} apiVersion: batch/v1 kind: CronJob metadata: @@ -27,16 +26,21 @@ spec: - -v - -u - $(DRYCC_STORAGE_ENDPOINT) - {{- include "builder.envs" . | indent 12 }} + {{- include "registry.envs" . | indent 12 }} containers: - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/registry:{{.Values.imageTag}} imagePullPolicy: {{.Values.imagePullPolicy}} name: drycc-registry-garbage-collect + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }} + {{- else }} args: - - /bin/boot - - garbage-collect - - $(DRYCC_REGISTRY_CONFIG) - - --dry-run - - --delete-untagged - {{- include "builder.envs" . | indent 12 }} -{{- end }} + - /usr/bin/env + - bash + - -ec + - | + # run garbage collect + start-registry garbage-collect ${DRYCC_REGISTRY_CONFIG} --dry-run --delete-untagged + {{- end }} + {{- include "registry.envs" . | indent 12 }} diff --git a/charts/registry/templates/registry-deployment.yaml b/charts/registry/templates/registry-deployment.yaml index 22b6561..dc362e3 100644 --- a/charts/registry/templates/registry-deployment.yaml +++ b/charts/registry/templates/registry-deployment.yaml @@ -1,4 +1,3 @@ -{{- if eq .Values.global.registryLocation "on-cluster" }} apiVersion: apps/v1 kind: Deployment metadata: @@ -41,8 +40,19 @@ spec: - name: drycc-registry image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/registry:{{.Values.imageTag}} imagePullPolicy: {{.Values.imagePullPolicy}} - {{- include "registry.limits" . | indent 8 }} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }} + {{- else }} + args: + - start-registry + {{- end }} + {{- with index .Values "resources" }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} {{- include "registry.envs" . | indent 8 }} + {{- if not .Values.diagnosticMode.enabled }} startupProbe: tcpSocket: port: 5000 @@ -69,6 +79,7 @@ spec: timeoutSeconds: 1 successThreshold: 1 failureThreshold: 5 + {{- end }} ports: - containerPort: 5000 name: http @@ -82,4 +93,3 @@ spec: volumes: - name: registry-storage emptyDir: {} -{{- end }} diff --git a/charts/registry/templates/registry-proxy-daemonset.yaml b/charts/registry/templates/registry-proxy-daemonset.yaml new file mode 100644 index 0000000..c83a364 --- /dev/null +++ b/charts/registry/templates/registry-proxy-daemonset.yaml @@ -0,0 +1,98 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: drycc-registry-proxy + labels: + heritage: drycc + annotations: + component.drycc.cc/version: {{ .Values.imageTag }} +spec: + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: drycc-registry-proxy + heritage: drycc + template: + metadata: + name: drycc-registry-proxy + labels: + heritage: drycc + app: drycc-registry-proxy + spec: + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsUser: 1001 + initContainers: + - name: drycc-registry-init + image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest + imagePullPolicy: {{.Values.imagePullPolicy}} + args: + - netcat + - -v + - -a + - $(DRYCC_REGISTRY_HOST) + env: + - name: "DRYCC_REGISTRY_HOST" + value: drycc-registry:5000 + containers: + - name: drycc-registry-proxy + image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/registry:{{.Values.imageTag}} + imagePullPolicy: {{.Values.imagePullPolicy}} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }} + {{- else }} + args: + - start-proxy + {{- end }} + {{- with index .Values "proxy" "resources" }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if not .Values.diagnosticMode.enabled }} + startupProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + livenessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + {{- end }} + env: + - name: "DRYCC_REGISTRY_HOST" + value: drycc-registry:5000 + - name: "DRYCC_REGISTRY_USERNAME" + valueFrom: + secretKeyRef: + name: registry-secret + key: username + - name: "DRYCC_REGISTRY_PASSWORD" + valueFrom: + secretKeyRef: + name: registry-secret + key: password + ports: + - containerPort: 8080 + hostPort: {{.Values.proxy.port}} diff --git a/charts/registry/templates/registry-secret.yaml b/charts/registry/templates/registry-secret.yaml index 301ceae..9fa12be 100644 --- a/charts/registry/templates/registry-secret.yaml +++ b/charts/registry/templates/registry-secret.yaml @@ -4,17 +4,16 @@ metadata: name: registry-secret labels: heritage: drycc - annotations: - drycc.cc/registry-location: "{{ .Values.global.registryLocation }}" type: Opaque data: - {{- if eq .Values.global.registryLocation "on-cluster" }} - host: {{ printf "drycc-registry.%s.svc.%s:5000" .Release.Namespace .Values.global.clusterDomain | b64enc }} - secret: {{ randAlphaNum 32 | b64enc }} - {{- else }} - host: {{ .Values.host | b64enc }} - organization: {{ .Values.organization | b64enc }} - {{- end }} - username: {{ if .Values.username | default "" | ne "" }}{{ .Values.username | b64enc }}{{ else }}{{ randAlphaNum 32 | b64enc }}{{ end }} - password: {{ if .Values.password | default "" | ne "" }}{{ .Values.password | b64enc }}{{ else }}{{ randAlphaNum 32 | b64enc }}{{ end }} + secret: {{ include "common.secrets.lookup" (dict "secret" "registry-secret" "key" "secret" "defaultValue" (randAlphaNum 32) "context" $) }} + username: {{ include "common.secrets.lookup" (dict "secret" "registry-secret" "key" "username" "defaultValue" (.Values.username | default (randAlphaNum 32)) "context" $) }} + password: {{ include "common.secrets.lookup" (dict "secret" "registry-secret" "key" "password" "defaultValue" (.Values.password | default (randAlphaNum 32)) "context" $) }} redirect: {{ .Values.redirect | b64enc }} + {{- if (.Values.storageEndpoint) }} + storage-bucket: {{ .Values.storageBucket | b64enc }} + storage-endpoint: {{ .Values.storageEndpoint | b64enc }} + storage-accesskey: {{ .Values.storageAccesskey | b64enc }} + storage-secretkey: {{ .Values.storageSecretkey | b64enc }} + storage-path-style: {{ .Values.storagePathStyle | b64enc }} + {{- end }} diff --git a/charts/registry/templates/registry-service-account.yaml b/charts/registry/templates/registry-service-account.yaml index 3766311..a3b2b78 100644 --- a/charts/registry/templates/registry-service-account.yaml +++ b/charts/registry/templates/registry-service-account.yaml @@ -1,8 +1,6 @@ -{{- if eq .Values.global.registryLocation "on-cluster" }} apiVersion: v1 kind: ServiceAccount metadata: name: drycc-registry labels: heritage: drycc -{{- end }} diff --git a/charts/registry/templates/registry-service.yaml b/charts/registry/templates/registry-service.yaml index 6b6d579..fbafcf2 100644 --- a/charts/registry/templates/registry-service.yaml +++ b/charts/registry/templates/registry-service.yaml @@ -1,9 +1,11 @@ -{{- if eq .Values.global.registryLocation "on-cluster" }} apiVersion: v1 kind: Service metadata: name: drycc-registry annotations: + prometheus.io/path: /metrics + prometheus.io/port: "9000" + prometheus.io/scrape: "true" {{- with .Values.service.annotations }} {{- toYaml . | nindent 4 }} {{- end }} @@ -13,5 +15,3 @@ spec: clusterIP: None selector: app: drycc-registry - sessionAffinity: ClientIP -{{- end }} diff --git a/charts/registry/values.yaml b/charts/registry/values.yaml index 99035c5..87a2895 100644 --- a/charts/registry/values.yaml +++ b/charts/registry/values.yaml @@ -2,8 +2,22 @@ imageOrg: "drycc" imagePullPolicy: "Always" imageTag: "canary" imageRegistry: "registry.drycc.cc" -# limitsCpu: "100m" -# limitsMemory: "50Mi" + +## Enable diagnostic mode +## +diagnosticMode: + ## @param diagnosticMode.enabled Enable diagnostic mode (all probes will be disabled and the command will be overridden) + ## + enabled: false + ## @param diagnosticMode.command Command to override all containers + ## + command: + - sleep + ## @param diagnosticMode.args Args to override all containers + ## + args: + - infinity + nodeAffinityPreset: key: "drycc.cc/node" type: "soft" @@ -23,11 +37,48 @@ podAntiAffinityPreset: # registry replicas replicas: 1 # registry storage redirect -redirect: false +redirect: "false" + +proxy: + # host port for the registry proxy in the daemonset + port: 5555 + resources: {} + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi + +resources: {} + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi concurrencyPolicy: "Replace" +# The following parameters will no longer use the built-in storage component. +storageBucket: "builder" +storageEndpoint: "" +storageAccesskey: "" +storageSecretkey: "" +storagePathStyle: "auto" + +storage: + enabled: true + # Service service: # Provide any additional service annotations - annotations: {} \ No newline at end of file + annotations: {} + +resources: {} + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi diff --git a/contrib/ci/s3.json b/contrib/ci/s3.json deleted file mode 100644 index c34dd33..0000000 --- a/contrib/ci/s3.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "identities": [{ - "name": "drycc", - "credentials": [{ - "accessKey": "$DRYCC_STORAGE_ACCESSKEY", - "secretKey": "$DRYCC_STORAGE_SECRETKEY" - }], - "actions": ["Admin", "Read", "List", "Tagging", "Write"] - }] -} \ No newline at end of file diff --git a/contrib/ci/start-s3.sh b/contrib/ci/start-s3.sh deleted file mode 100755 index 55b5092..0000000 --- a/contrib/ci/start-s3.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -eval "cat </etc/seaweedfs/s3.json -$( cat /tmp/weed/s3.json ) -EOF -" 2> /dev/null - -weed server -dir=/data -s3 -s3.config=/etc/seaweedfs/s3.json diff --git a/contrib/ci/test.sh b/contrib/ci/test.sh index d2fc18e..6be70b3 100755 --- a/contrib/ci/test.sh +++ b/contrib/ci/test.sh @@ -2,51 +2,73 @@ set -eoxf pipefail -BASE_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") DRYCC_STORAGE_ACCESSKEY=f4c4281665bc11ee8e0400163e04a9cd DRYCC_STORAGE_SECRETKEY=f4c4281665bc11ee8e0400163e04a9cd -STORAGE_JOB=$(podman run -d --entrypoint init-stack -p 8333:8333 \ - -v "${BASE_DIR}":/tmp/weed \ - -e DRYCC_STORAGE_ACCESSKEY="${DRYCC_STORAGE_ACCESSKEY}" \ - -e DRYCC_STORAGE_SECRETKEY="${DRYCC_STORAGE_SECRETKEY}" \ - "${DEV_REGISTRY}"/drycc/storage:canary /tmp/weed/start-s3.sh) +STORAGE_JOB=$(podman run -d --rm --entrypoint init-stack \ + -e RUSTFS_ACCESS_KEY="${DRYCC_STORAGE_ACCESSKEY}" \ + -e RUSTFS_SECRET_KEY="${DRYCC_STORAGE_SECRETKEY}" \ + "${DEV_REGISTRY}"/drycc/storage:canary rustfs /data) # wait for port STORAGE_IP=$(podman inspect --format "{{ .NetworkSettings.IPAddress }}" "${STORAGE_JOB}") -echo -e "\\033[32m---> Waitting for ${STORAGE_IP}:8333\\033[0m" -wait-for-port --host="${STORAGE_IP}" 8333 -echo -e "\\033[32m---> S3 service ${STORAGE_IP}:8333 ready...\\033[0m" +echo -e "\\033[32m---> Waitting for ${STORAGE_IP}:9000\\033[0m" +wait-for-port --host="${STORAGE_IP}" 9000 +echo -e "\\033[32m---> S3 service ${STORAGE_IP}:9000 ready...\\033[0m" podman logs "${STORAGE_JOB}" -JOB=$(podman run -d \ - -e REGISTRY_HTTP_SECRET=drycc \ +REGISTRY_JOB=$(podman run -d --rm \ -e DRYCC_REGISTRY_REDIRECT=false \ -e DRYCC_REGISTRY_USERNAME=admin \ -e DRYCC_REGISTRY_PASSWORD=admin \ - -e DRYCC_STORAGE_LOOKUP=path \ -e DRYCC_STORAGE_BUCKET=registry \ - -e DRYCC_STORAGE_ENDPOINT="http://${STORAGE_IP}:8333" \ + -e DRYCC_STORAGE_ENDPOINT="http://${STORAGE_IP}:9000" \ -e DRYCC_STORAGE_ACCESSKEY="${DRYCC_STORAGE_ACCESSKEY}" \ -e DRYCC_STORAGE_SECRETKEY="${DRYCC_STORAGE_SECRETKEY}" \ - "$1") + -e DRYCC_STORAGE_PATH_STYLE=true \ + "$1" start-registry) # shellcheck disable=SC2317 function clean_before_exit { # delay before exiting, so stdout/stderr flushes through the logging system - podman kill "${JOB}" + podman kill "${REGISTRY_JOB}" podman kill "${STORAGE_JOB}" - podman rm -f "${JOB}" "${STORAGE_JOB}" + podman kill "${PROXY_JOB}" } trap clean_before_exit EXIT # let the registry run for a few seconds -REGISTRY_IP=$(podman inspect --format "{{ .NetworkSettings.IPAddress }}" "${JOB}") +REGISTRY_IP=$(podman inspect --format "{{ .NetworkSettings.IPAddress }}" "${REGISTRY_JOB}") echo -e "\\033[32m---> Waitting for ${REGISTRY_IP}:5000\\033[0m" wait-for-port --host="${REGISTRY_IP}" 5000 echo -e "\\033[32m---> S3 service ${REGISTRY_IP}:5000 ready...\\033[0m" + +# proxy job +PROXY_JOB=$(podman run -d \ + -p 15555:8080 \ + -e DRYCC_REGISTRY_HOST="${REGISTRY_IP}:5000" \ + -e DRYCC_REGISTRY_USERNAME=admin \ + -e DRYCC_REGISTRY_PASSWORD=admin \ + "$1" start-proxy) + +# let the registry proxy run for a few seconds +REGISTRY_PROXY_IP=$(podman inspect --format "{{ .NetworkSettings.IPAddress }}" "${PROXY_JOB}") +echo -e "\\033[32m---> Waitting for ${REGISTRY_PROXY_IP}:8080\\033[0m" +wait-for-port --host="${REGISTRY_PROXY_IP}" 8080 +echo -e "\\033[32m---> S3 service ${REGISTRY_PROXY_IP}:8080 ready...\\033[0m" + # check that the registry is still up -podman tag "$1" "${REGISTRY_IP}:5000/registry:canary" -echo admin | podman login "${REGISTRY_IP}:5000" --tls-verify=false --username admin --password-stdin > /dev/null 2>&1 -podman push "${REGISTRY_IP}:5000/registry:canary" --tls-verify=false +http_status_code=$(curl -X GET -s -o /dev/null -w "%{http_code}" "http://${REGISTRY_PROXY_IP}:8080/v2/") +if [ "$http_status_code" != "200" ]; then + echo "Expected http status code: 200, actual: ${http_status_code}" + exit 1 +fi + +http_status_code=$(curl -X POST -s -o /dev/null -w "%{http_code}" "http://${REGISTRY_PROXY_IP}:8080/v2/") +if [ "$http_status_code" != "403" ]; then + echo "Expected http status code: 403, actual: ${http_status_code}" + exit 1 +fi + +echo -e "\\033[32m---> All test success...\\033[0m" \ No newline at end of file diff --git a/main.go b/main.go index 07ff818..27b534c 100644 --- a/main.go +++ b/main.go @@ -14,11 +14,11 @@ const ( registryHtpasswd = "/opt/drycc/registry/etc/htpasswd" registryConfigEnvVar = "DRYCC_REGISTRY_CONFIG" registryRedirectEnvVar = "DRYCC_REGISTRY_REDIRECT" - storageLookupEnvVar = "DRYCC_STORAGE_LOOKUP" storageBucketEnvVar = "DRYCC_STORAGE_BUCKET" storageEndpointEnvVar = "DRYCC_STORAGE_ENDPOINT" storageAccesskeyEnvVar = "DRYCC_STORAGE_ACCESSKEY" storageSecretkeyEnvVar = "DRYCC_STORAGE_SECRETKEY" + storagePathStyleEnvVar = "DRYCC_STORAGE_PATH_STYLE" defaultCommand = "serve" ) @@ -40,7 +40,7 @@ func main() { os.Setenv("REGISTRY_STORAGE_S3_SECRETKEY", os.Getenv(storageSecretkeyEnvVar)) os.Setenv("REGISTRY_STORAGE_S3_BUCKET", os.Getenv(storageBucketEnvVar)) - if os.Getenv(storageLookupEnvVar) == "path" { + if os.Getenv(storagePathStyleEnvVar) == "true" { os.Setenv("REGISTRY_STORAGE_S3_FORCEPATHSTYLE", "true") } @@ -57,16 +57,18 @@ func main() { os.Setenv("REGISTRY_VALIDATION_DISABLED", "true") os.Setenv("REGISTRY_STORAGE_S3_ROOTDIRECTORY", "/registry") - // run /bin/init_registry + // run /bin/init-registry os.Setenv("REGISTRY_AUTH", "htpasswd") os.Setenv("REGISTRY_AUTH_HTPASSWD_REALM", "basic-realm") os.Setenv("REGISTRY_AUTH_HTPASSWD_PATH", registryHtpasswd) - cmd := exec.Command("/bin/init_registry") + cmd := exec.Command("/bin/init-registry") cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { log.Fatal("Error creating the registry bucket: ", err) } + // avoid conflicts with env variables + os.Unsetenv("REGISTRY_VERSION") if len(os.Args) > 1 { cmd = exec.Command(registryBinary, os.Args[1:]...) } else { diff --git a/rootfs/bin/init-registry b/rootfs/bin/init-registry new file mode 100755 index 0000000..4d05fc8 --- /dev/null +++ b/rootfs/bin/init-registry @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -e + + +mkdir -p ~/.config/rclone +touch ~/.config/rclone/rclone.conf +rclone config create storage s3 \ + provider=Other \ + access_key_id="${DRYCC_STORAGE_ACCESSKEY}" \ + secret_access_key="${DRYCC_STORAGE_SECRETKEY}" \ + endpoint="${DRYCC_STORAGE_ENDPOINT}" \ + force_path_style="${DRYCC_STORAGE_PATH_STYLE:-true}" --no-output + +if ! rclone lsd storage: > /dev/null 2>&1; then + sleep 9s + echo "waiting for object storage to become ready..." +fi + +rclone mkdir "storage:${DRYCC_STORAGE_BUCKET}" + +htpasswd -Bbn "${DRYCC_REGISTRY_USERNAME}" "${DRYCC_REGISTRY_PASSWORD}" > "${REGISTRY_AUTH_HTPASSWD_PATH}" +echo "create ${REGISTRY_AUTH_HTPASSWD_PATH} success" diff --git a/rootfs/bin/init_registry b/rootfs/bin/init_registry deleted file mode 100755 index a3a6041..0000000 --- a/rootfs/bin/init_registry +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -e - -mc config host add storage \ - "${DRYCC_STORAGE_ENDPOINT}" \ - "${DRYCC_STORAGE_ACCESSKEY}" \ - "${DRYCC_STORAGE_SECRETKEY}" \ - --lookup "${DRYCC_STORAGE_LOOKUP}" \ - --api s3v4 - -has_bucket(){ - mc ls storage -json|jq -r '.key'|grep -w "${DRYCC_STORAGE_BUCKET}" -} - -if [ -z "$(has_bucket)" ] ;then - mc mb storage/"${DRYCC_STORAGE_BUCKET}" - if [ -z "$(has_bucket)" ] ;then - echo "create bucket ${DRYCC_STORAGE_BUCKET} error" - exit 1 - fi -fi -echo "create bucket ${DRYCC_STORAGE_BUCKET} success" - -htpasswd -Bbn "${DRYCC_REGISTRY_USERNAME}" "${DRYCC_REGISTRY_PASSWORD}" > "${REGISTRY_AUTH_HTPASSWD_PATH}" -echo "create ${REGISTRY_AUTH_HTPASSWD_PATH} success" \ No newline at end of file diff --git a/rootfs/bin/start-proxy b/rootfs/bin/start-proxy new file mode 100755 index 0000000..5354551 --- /dev/null +++ b/rootfs/bin/start-proxy @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +REGISTRY_HOST=${DRYCC_REGISTRY_HOST:?no host} +AUTHORIZATION=$(echo -ne "${DRYCC_REGISTRY_USERNAME:? no username}":"${DRYCC_REGISTRY_PASSWORD:? no password}" | base64 -w 0) + +cat /opt/drycc/nginx/conf/registry.conf.tpl > /opt/drycc/nginx/conf/registry.conf +sed -i "s#%REGISTRY_HOST%#${REGISTRY_HOST}#g" /opt/drycc/nginx/conf/registry.conf +sed -i "s#%AUTHORIZATION%#${AUTHORIZATION}#g" /opt/drycc/nginx/conf/registry.conf + +# wait for registry to come online +while ! curl -sS "$REGISTRY_HOST" &>/dev/null; do + echo "waiting for the registry (%s) to come online..." + echo "$REGISTRY_HOST" + sleep 1 +done + +echo "starting registry-proxy..." +exec nginx -g "daemon off;" diff --git a/rootfs/config-example.yml b/rootfs/config-example.yml index 3277f9a..a55566c 100644 --- a/rootfs/config-example.yml +++ b/rootfs/config-example.yml @@ -9,6 +9,11 @@ storage: rootdirectory: /var/lib/registry http: addr: :5000 + debug: + addr: :9000 + prometheus: + enabled: true + path: /metrics headers: X-Content-Type-Options: [nosniff] health: diff --git a/rootfs/opt/drycc/nginx/conf/nginx.conf b/rootfs/opt/drycc/nginx/conf/nginx.conf new file mode 100644 index 0000000..bd3f31d --- /dev/null +++ b/rootfs/opt/drycc/nginx/conf/nginx.conf @@ -0,0 +1,22 @@ +worker_processes 1; + +error_log /dev/stderr warn; +pid /opt/drycc/nginx/logs/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /opt/drycc/nginx/conf/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /dev/stdout main; + sendfile on; + keepalive_timeout 65; + include /opt/drycc/nginx/conf/registry.conf; +} diff --git a/rootfs/opt/drycc/nginx/conf/registry.conf.tpl b/rootfs/opt/drycc/nginx/conf/registry.conf.tpl new file mode 100644 index 0000000..627f2f2 --- /dev/null +++ b/rootfs/opt/drycc/nginx/conf/registry.conf.tpl @@ -0,0 +1,24 @@ +upstream container-registry { + server %REGISTRY_HOST%; +} + +server { + listen 8080; + server_name localhost; + # disable any limits to avoid HTTP 413 for large image uploads + client_max_body_size 0; + # required to avoid HTTP 411: see Issue #1486 (https://github.com/moby/moby/issues/1486) + chunked_transfer_encoding on; + location / { + proxy_pass http://container-registry; + proxy_set_header Host $http_host; # required for container client's sake + proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + proxy_set_header Authorization "Basic %AUTHORIZATION%"; + limit_except GET HEAD OPTIONS { + deny all; + } + } +} \ No newline at end of file