Skip to content

Commit 7b889aa

Browse files
duanhongyiduanhongyi
committed
chore(registry): add registry proxy
1 parent d159e8e commit 7b889aa

12 files changed

Lines changed: 236 additions & 21 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
rootfs/opt/
21
contrib/ci/tmp/
32
.vscode/
3+
rootfs/opt/registry/sbin

‎Dockerfile‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ ARG LDFLAGS
55
ADD . /workspace
66
RUN export GO111MODULE=on \
77
&& cd /workspace \
8-
&& CGO_ENABLED=0 init-stack go build -ldflags "${LDFLAGS}" -o /bin/boot main.go \
9-
&& upx -9 --brute /bin/boot
8+
&& CGO_ENABLED=0 init-stack go build -ldflags "${LDFLAGS}" -o /bin/start-registry main.go \
9+
&& upx -9 --brute /bin/start-registry
1010

1111

1212
FROM registry.drycc.cc/drycc/base:${CODENAME}
@@ -16,18 +16,16 @@ ENV DRYCC_UID=1001 \
1616
DRYCC_HOME_DIR=/var/lib/registry \
1717
JQ_VERSION="1.7.1" \
1818
MC_VERSION="2025.04.03.17.07.56" \
19+
NGINX_VERSION="1.25.1" \
1920
REGISTRY_VERSION="3.0.0"
2021

21-
COPY rootfs/bin/ /bin/
22-
COPY --from=build /bin/boot /bin/boot
23-
2422
RUN groupadd drycc --gid ${DRYCC_GID} \
2523
&& useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} \
2624
&& install-packages apache2-utils \
2725
&& install-stack jq $JQ_VERSION \
2826
&& install-stack mc $MC_VERSION \
27+
&& install-stack nginx ${NGINX_VERSION} \
2928
&& install-stack registry $REGISTRY_VERSION \
30-
&& chmod +x /bin/init_registry \
3129
&& rm -rf \
3230
/usr/share/doc \
3331
/usr/share/man \
@@ -42,12 +40,15 @@ RUN groupadd drycc --gid ${DRYCC_GID} \
4240
/usr/lib/`echo $(uname -m)`-linux-gnu/gconv/IBM* \
4341
/usr/lib/`echo $(uname -m)`-linux-gnu/gconv/EBC* \
4442
&& mkdir -p /usr/share/man/man{1..8} \
45-
&& chown -R ${DRYCC_UID}:${DRYCC_GID} ${DRYCC_HOME_DIR}
43+
&& chown -R ${DRYCC_UID}:${DRYCC_GID} /opt/drycc
4644

45+
COPY --from=build /bin/start-registry /bin/start-registry
46+
COPY --chown=${DRYCC_UID}:${DRYCC_GID} rootfs/bin/ /bin/
47+
COPY --chown=${DRYCC_UID}:${DRYCC_GID} rootfs/opt/drycc/nginx /opt/drycc/nginx
4748
COPY --chown=${DRYCC_UID}:${DRYCC_GID} rootfs/config-example.yml /opt/drycc/registry/etc/config.yml
49+
4850
ENV DRYCC_REGISTRY_CONFIG /opt/drycc/registry/etc/config.yml
4951

5052
USER ${DRYCC_UID}
5153
VOLUME ["${DRYCC_HOME_DIR}"]
52-
CMD ["/bin/boot"]
5354
EXPOSE 5000

‎README.md‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,15 @@ We welcome your input! If you have feedback, please submit an [issue][issues]. I
1212

1313
# About
1414

15-
The registry is a [Container registry](https://github.com/distribution/distribution) component for use in Kubernetes. While it's intended for use inside of the Drycc open source [PaaS](https://en.wikipedia.org/wiki/Platform_as_a_service), it's flexible enough to be used as a standalone pod on any Kubernetes cluster.
15+
Registry consists of two components, namely the proxy component and the registry component.
16+
17+
## Proxy
18+
19+
The proxy component is a proxy deployed on every Kubernetes worker node, proxying all requests to the Drycc Workflow [registry][registry]. This allows the worker nodes daemons to communicate to the registry over localhost, bypassing the need for adding the `--insecure-registry` flag to the daemons.
20+
21+
## Registry
22+
23+
The registry component is a [Container registry](https://github.com/distribution/distribution) component for use in Kubernetes. While it's intended for use inside of the Drycc open source [PaaS](https://en.wikipedia.org/wiki/Platform_as_a_service), it's flexible enough to be used as a standalone pod on any Kubernetes cluster.
1624

1725
If you decide to use this component standalone, you can host your own Container registry in your own Kubernetes cluster.
1826

‎charts/registry/templates/registry-deployment.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ spec:
4444
{{- if .Values.diagnosticMode.enabled }}
4545
command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }}
4646
args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }}
47+
{{- else }}
48+
args:
49+
- start-registry
4750
{{- end }}
4851
{{- include "registry.limits" . | indent 8 }}
4952
{{- include "registry.envs" . | indent 8 }}

‎charts/registry/templates/registry-proxy-daemonset.yaml‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
{{- if eq .Values.global.registryLocation "on-cluster" }}
2+
apiVersion: apps/v1
3+
kind: DaemonSet
4+
metadata:
5+
name: drycc-registry-proxy
6+
labels:
7+
heritage: drycc
8+
annotations:
9+
component.drycc.cc/version: {{ .Values.imageTag }}
10+
spec:
11+
updateStrategy:
12+
type: RollingUpdate
13+
selector:
14+
matchLabels:
15+
app: drycc-registry-proxy
16+
heritage: drycc
17+
template:
18+
metadata:
19+
name: drycc-registry-proxy
20+
labels:
21+
heritage: drycc
22+
app: drycc-registry-proxy
23+
spec:
24+
securityContext:
25+
fsGroup: 1001
26+
runAsGroup: 1001
27+
runAsUser: 1001
28+
initContainers:
29+
- name: drycc-registry-init
30+
image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest
31+
imagePullPolicy: {{.Values.imagePullPolicy}}
32+
args:
33+
- netcat
34+
- -v
35+
- -a
36+
- $(DRYCC_REGISTRY_HOST)
37+
env:
38+
- name: "DRYCC_REGISTRY_HOST"
39+
valueFrom:
40+
secretKeyRef:
41+
name: registry-secret
42+
key: host
43+
containers:
44+
- name: drycc-registry-proxy
45+
image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/registry:{{.Values.imageTag}}
46+
imagePullPolicy: {{.Values.imagePullPolicy}}
47+
{{- if .Values.diagnosticMode.enabled }}
48+
command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }}
49+
args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }}
50+
{{- else }}
51+
args:
52+
- start-proxy
53+
{{- end }}
54+
{{- if or (.Values.limitsCpu) (.Values.limitsMemory)}}
55+
resources:
56+
limits:
57+
{{- if (.Values.limitsCpu) }}
58+
cpu: {{.Values.limitsCpu}}
59+
{{- end}}
60+
{{- if (.Values.limitsMemory) }}
61+
memory: {{.Values.limitsMemory}}
62+
{{- end}}
63+
{{- end}}
64+
{{- if not .Values.diagnosticMode.enabled }}
65+
startupProbe:
66+
httpGet:
67+
path: /
68+
port: 8080
69+
initialDelaySeconds: 30
70+
periodSeconds: 5
71+
timeoutSeconds: 1
72+
successThreshold: 1
73+
failureThreshold: 5
74+
livenessProbe:
75+
httpGet:
76+
path: /
77+
port: 8080
78+
initialDelaySeconds: 30
79+
periodSeconds: 5
80+
timeoutSeconds: 1
81+
successThreshold: 1
82+
failureThreshold: 5
83+
readinessProbe:
84+
httpGet:
85+
path: /
86+
port: 8080
87+
initialDelaySeconds: 30
88+
periodSeconds: 5
89+
timeoutSeconds: 1
90+
successThreshold: 1
91+
failureThreshold: 5
92+
{{- end }}
93+
env:
94+
- name: "DRYCC_REGISTRY_HOST"
95+
valueFrom:
96+
secretKeyRef:
97+
name: registry-secret
98+
key: host
99+
- name: "DRYCC_REGISTRY_USERNAME"
100+
valueFrom:
101+
secretKeyRef:
102+
name: registry-secret
103+
key: username
104+
- name: "DRYCC_REGISTRY_PASSWORD"
105+
valueFrom:
106+
secretKeyRef:
107+
name: registry-secret
108+
key: password
109+
ports:
110+
- containerPort: 8080
111+
hostPort: {{.Values.proxyPort}}
112+
{{- end }}

‎charts/registry/values.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ podAntiAffinityPreset:
4040
replicas: 1
4141
# registry storage redirect
4242
redirect: "false"
43+
# host port for the registry proxy in the daemonset
44+
proxyPort: 5555
4345

4446
concurrencyPolicy: "Replace"
4547

‎contrib/ci/test.sh‎

Lines changed: 34 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ DRYCC_STORAGE_ACCESSKEY=f4c4281665bc11ee8e0400163e04a9cd
66
DRYCC_STORAGE_SECRETKEY=f4c4281665bc11ee8e0400163e04a9cd
77

88

9-
STORAGE_JOB=$(podman run -d --entrypoint init-stack \
9+
STORAGE_JOB=$(podman run -d --rm --entrypoint init-stack \
1010
-e MINIO_ROOT_USER="${DRYCC_STORAGE_ACCESSKEY}" \
1111
-e MINIO_ROOT_PASSWORD="${DRYCC_STORAGE_SECRETKEY}" \
1212
"${DEV_REGISTRY}"/drycc/storage:canary minio server /data)
@@ -18,7 +18,7 @@ wait-for-port --host="${STORAGE_IP}" 9000
1818
echo -e "\\033[32m---> S3 service ${STORAGE_IP}:9000 ready...\\033[0m"
1919
podman logs "${STORAGE_JOB}"
2020

21-
JOB=$(podman run -d \
21+
REGISTRY_JOB=$(podman run -d --rm \
2222
-e REGISTRY_HTTP_SECRET=drycc \
2323
-e DRYCC_REGISTRY_REDIRECT=false \
2424
-e DRYCC_REGISTRY_USERNAME=admin \
@@ -28,23 +28,48 @@ JOB=$(podman run -d \
2828
-e DRYCC_STORAGE_ENDPOINT="http://${STORAGE_IP}:9000" \
2929
-e DRYCC_STORAGE_ACCESSKEY="${DRYCC_STORAGE_ACCESSKEY}" \
3030
-e DRYCC_STORAGE_SECRETKEY="${DRYCC_STORAGE_SECRETKEY}" \
31-
"$1")
31+
"$1" start-registry)
3232

3333
# shellcheck disable=SC2317
3434
function clean_before_exit {
3535
# delay before exiting, so stdout/stderr flushes through the logging system
36-
podman kill "${JOB}"
36+
podman kill "${REGISTRY_JOB}"
3737
podman kill "${STORAGE_JOB}"
38-
podman rm -f "${JOB}" "${STORAGE_JOB}"
38+
podman kill "${PROXY_JOB}"
3939
}
4040
trap clean_before_exit EXIT
4141

4242
# let the registry run for a few seconds
43-
REGISTRY_IP=$(podman inspect --format "{{ .NetworkSettings.IPAddress }}" "${JOB}")
43+
REGISTRY_IP=$(podman inspect --format "{{ .NetworkSettings.IPAddress }}" "${REGISTRY_JOB}")
4444
echo -e "\\033[32m---> Waitting for ${REGISTRY_IP}:5000\\033[0m"
4545
wait-for-port --host="${REGISTRY_IP}" 5000
4646
echo -e "\\033[32m---> S3 service ${REGISTRY_IP}:5000 ready...\\033[0m"
47+
48+
# proxy job
49+
PROXY_JOB=$(podman run -d \
50+
-p 15555:8080 \
51+
-e DRYCC_REGISTRY_HOST="${REGISTRY_IP}:5000" \
52+
-e DRYCC_REGISTRY_USERNAME=admin \
53+
-e DRYCC_REGISTRY_PASSWORD=admin \
54+
"$1" start-proxy)
55+
56+
# let the registry proxy run for a few seconds
57+
REGISTRY_PROXY_IP=$(podman inspect --format "{{ .NetworkSettings.IPAddress }}" "${PROXY_JOB}")
58+
echo -e "\\033[32m---> Waitting for ${REGISTRY_PROXY_IP}:8080\\033[0m"
59+
wait-for-port --host="${REGISTRY_PROXY_IP}" 8080
60+
echo -e "\\033[32m---> S3 service ${REGISTRY_PROXY_IP}:8080 ready...\\033[0m"
61+
4762
# check that the registry is still up
48-
podman tag "$1" "${REGISTRY_IP}:5000/registry:canary"
49-
echo admin | podman login "${REGISTRY_IP}:5000" --tls-verify=false --username admin --password-stdin > /dev/null 2>&1
50-
podman push "${REGISTRY_IP}:5000/registry:canary" --tls-verify=false
63+
http_status_code=$(curl -X GET -s -o /dev/null -w "%{http_code}" "http://${REGISTRY_PROXY_IP}:8080/v2/")
64+
if [ "$http_status_code" != "200" ]; then
65+
echo "Expected http status code: 200, actual: ${http_status_code}"
66+
exit 1
67+
fi
68+
69+
http_status_code=$(curl -X POST -s -o /dev/null -w "%{http_code}" "http://${REGISTRY_PROXY_IP}:8080/v2/")
70+
if [ "$http_status_code" != "403" ]; then
71+
echo "Expected http status code: 403, actual: ${http_status_code}"
72+
exit 1
73+
fi
74+
75+
echo -e "\\033[32m---> All test success...\\033[0m"

‎main.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,11 +57,11 @@ func main() {
5757
os.Setenv("REGISTRY_VALIDATION_DISABLED", "true")
5858
os.Setenv("REGISTRY_STORAGE_S3_ROOTDIRECTORY", "/registry")
5959

60-
// run /bin/init_registry
60+
// run /bin/init-registry
6161
os.Setenv("REGISTRY_AUTH", "htpasswd")
6262
os.Setenv("REGISTRY_AUTH_HTPASSWD_REALM", "basic-realm")
6363
os.Setenv("REGISTRY_AUTH_HTPASSWD_PATH", registryHtpasswd)
64-
cmd := exec.Command("/bin/init_registry")
64+
cmd := exec.Command("/bin/init-registry")
6565
cmd.Stdout = os.Stdout
6666
cmd.Stderr = os.Stderr
6767
if err := cmd.Run(); err != nil {

rootfs/bin/init_registry renamed to rootfs/bin/init-registry

File renamed without changes.

‎rootfs/bin/start-proxy‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env bash
2+
3+
REGISTRY_HOST=${DRYCC_REGISTRY_HOST:?no host}
4+
AUTHORIZATION=$(echo -ne "${DRYCC_REGISTRY_USERNAME:? no username}":"${DRYCC_REGISTRY_PASSWORD:? no password}" | base64 -w 0)
5+
6+
cat /opt/drycc/nginx/conf/registry.conf.tpl > /opt/drycc/nginx/conf/registry.conf
7+
sed -i "s#%REGISTRY_HOST%#${REGISTRY_HOST}#g" /opt/drycc/nginx/conf/registry.conf
8+
sed -i "s#%AUTHORIZATION%#${AUTHORIZATION}#g" /opt/drycc/nginx/conf/registry.conf
9+
10+
# wait for registry to come online
11+
while ! curl -sS "$REGISTRY_HOST" &>/dev/null; do
12+
echo "waiting for the registry (%s) to come online..."
13+
echo "$REGISTRY_HOST"
14+
sleep 1
15+
done
16+
17+
echo "starting registry-proxy..."
18+
exec nginx -g "daemon off;"

0 commit comments

Comments
 (0)