chore(controller): CELERY_BROKER use rabbitmq and modify celery-deployment cronjob

Author: lijianguo

charts/controller/templates/controller-celery.yaml

@@ -21,7 +21,6 @@ spec:
       labels:
         app: drycc-controller-celery
     spec:
-      serviceAccount: drycc-controller-celery
       containers:
         - name: drycc-controller-celery-high
           image: {{.Values.image_registry}}{{.Values.org}}/controller:{{.Values.image_tag}}
@@ -62,4 +61,4 @@ spec:
           {{- include "controller.envs" . | indent 10 }}
           {{- include "controller.volumeMounts" . | indent 10 }}
           imagePullPolicy: {{.Values.image_pull_policy}}
-      {{- include "controller.volumes" . | indent 6 }}
+      {{- include "controller.volumes" . | indent 6 }}

charts/controller/templates/controller-cronjob-daily.yaml

@@ -23,7 +23,7 @@ spec:
               - /bin/bash
               - -c
             args:
-              - python /app/manage.py push_data_to_influxdb
+              - python /app/manage.py load_db_state_to_k8s
             {{- include "controller.envs" . | indent 12 }}
             volumeMounts:
             - mountPath: /etc/slugrunner
@@ -79,4 +79,3 @@ spec:
             {{- include "controller.envs" . | indent 12 }}
             {{- include "controller.volumeMounts" . | indent 12 }}
           {{- include "controller.volumes" . | indent 10 }}
-

rootfs/Dockerfile.test

@@ -29,6 +29,7 @@ RUN echo https://dl-cdn.alpinelinux.org/alpine/edge/testing >>/etc/apk/repositor
     shadow \
     postgresql \
     redis \
+    rabbitmq-server \
   && mkdir -p /run/postgresql $PGDATA \
   && chown -R postgres:postgres /run/postgresql $PGDATA \
   && apk del .build-deps \

rootfs/api/settings/production.py

@@ -527,8 +527,7 @@
 CELERY_TASK_TIME_LIMIT = 30 * 60
 CELERYD_MAX_TASKS_PER_CHILD = 200
 CELERY_TASK_RESULT_EXPIRES = 24 * 60 * 60
-CELERY_BROKER_URL ='redis://:{}@{}'.format(DRYCC_REDIS_PASSWORD, DRYCC_REDIS_ADDRS[0])  # noqa
-CELERY_RESULT_BACKEND = 'redis://:{}@{}'.format(DRYCC_REDIS_PASSWORD, DRYCC_REDIS_ADDRS[0])  # noqa
+CELERY_BROKER_URL = os.environ.get('DRYCC_RABBITMQ_URL', 'amqp://guest:guest@127.0.0.1:5672/')  # noqa
 CELERY_CACHE_BACKEND = 'django-cache'
 CELERY_DEFAULT_QUEUE = 'priority.middle'
 

rootfs/bin/test-unit

@@ -12,6 +12,12 @@ function start_redis() {
     cd -
 }
 
+function start_rabbitmq() {
+    cd /tmp
+    nohup rabbitmq-server > /var/log/rabbitmq.log 2>&1 &
+    cd -
+}
+
 function start_celery() {
     cd /tmp
     nohup su-exec nobody celery -A api worker -Q priority.low,priority.middle,priority.high --loglevel=info > /var/log/celery.log 2>&1 &
@@ -20,6 +26,7 @@ function start_celery() {
 
 su-exec postgres pg_ctl -D "$PGDATA" start
 start_redis
+start_rabbitmq
 start_celery
 python3 manage.py check
 coverage run manage.py test --settings=api.settings.testing --noinput api scheduler.tests