chore(controller): add measure loalancers

Author: duanhongyi

charts/controller/templates/_helpers.tpl

@@ -129,6 +129,26 @@ env:
     secretKeyRef:
       name: redis-creds
       key: password
+{{- if eq .Values.global.rabbitmqLocation "off-cluster" }}
+- name: "DRYCC_PROMETHEUS_URL"
+  valueFrom:
+    secretKeyRef:
+      name: prometheus-creds
+      key: url
+{- else }
+- name: "DRYCC_PROMETHEUS_USERNAME"
+  valueFrom:
+    secretKeyRef:
+      name: prometheus-creds
+      key: username
+- name: "DRYCC_PROMETHEUS_PASSWORD"
+  valueFrom:
+    secretKeyRef:
+      name: prometheus-creds
+      key: password
+- name: "DRYCC_PROMETHEUS_URL"
+  value: "http://$(DRYCC_PROMETHEUS_USERNAME):$(DRYCC_PROMETHEUS_PASSWORD)@drycc-prometheus.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:9090//api/v1/query"
+{{- end }}
 {{- if (.Values.rabbitmqUrl) }}
 - name: DRYCC_RABBITMQ_URL
   valueFrom:

charts/controller/templates/controller-cronjob-hourly.yaml

@@ -42,3 +42,11 @@ spec:
             - -c
             - python -u /workspace/manage.py measure_networks
             {{- include "controller.envs" . | indent 12 }}
+          - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
+            imagePullPolicy: {{.Values.pull_policy}}
+            name: drycc-controller-measure-loadbalancer
+            args:
+            - /bin/bash
+            - -c
+            - python -u /workspace/manage.py measure_loadbalancers
+            {{- include "controller.envs" . | indent 12 }}

rootfs/api/management/commands/measure_loadbalancers.py

@@ -0,0 +1,50 @@
+import uuid
+import time
+import logging
+from django.utils import timezone
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from api import monitor
+from api.models.app import App
+from api.tasks import send_measurements
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """Management command for push data to manager"""
+
+    def _measure_loadbalancers(self, app_map, timestamp):
+        stop = timestamp - (timestamp % 3600)
+        start = stop - 3600
+        loadbalancers = []
+        for item in monitor.query_loadbalancer(app_map.keys(), start, stop):
+            name = item["ip"]
+            namespace = item["namespace"]
+            owner_id = app_map[namespace].owner_id
+            loadbalancers.append({
+                "app_id":  str(app_map[namespace].uuid),
+                "owner": owner_id,
+                "name": name,
+                "type": "loadbalancer",
+                "unit": "number",
+                "usage": 1,
+                "timestamp": start
+            })
+        send_measurements.delay(loadbalancers)
+
+    def handle(self, *args, **options):
+        if settings.WORKFLOW_MANAGER_URL and settings.DRYCC_PROMETHEUS_URL:
+            timestamp = int(time.time())
+            task_id = uuid.uuid4().hex
+            logger.info(f"pushing {task_id} limits to workflow_manager when {timezone.now()}")
+            app_map = {}
+            for app in App.objects.all():
+                app_map[app.id] = app
+                if len(app_map) % 1000 == 0:
+                    self._measure_loadbalancers(app_map, timestamp)
+                    app_map = {}
+            if len(app_map) > 0:
+                self._measure_loadbalancers(app_map, timestamp)
+            logger.info(f"pushed {task_id} limits to workflow_manager when {timezone.now()}")
+            self.stdout.write("done")

rootfs/api/monitor.py

@@ -1,6 +1,8 @@
-from typing import Iterator
+import requests
+from typing import Iterator, Dict
 from contextlib import closing
 from django.db import connections
+from django.conf import settings
 
 
 query_network_flow_sql_tpl = """
@@ -101,6 +103,21 @@
 """
 
 
+query_loadbalancer_promql_tpl = """
+max_over_time(kube_service_status_load_balancer_ingress{namespace=~"%s"}[60m])
+"""
+
+
+def query_loadbalancer(namespaces: Iterator[str],
+                        start: int, stop: int) -> Iterator[Dict[str, str]]:
+    promql = query_loadbalancer_promql_tpl % "|".join(namespaces)
+    params = {"query": promql, "start": start, "end": stop}
+    response = requests.get(settings.DRYCC_PROMETHEUS_URL, params=params)
+    if response.status_code != 200:
+        return StopIteration
+    yield from (metric["metric"] for metric in response.json()["data"]["result"])
+
+
 def query_network_flow(namespaces: Iterator[str],
                        start: int, stop: int) -> Iterator[tuple[str, str, int, int]]:
     with closing(connections['monitor'].cursor()) as cursor:

rootfs/api/settings/production.py

@@ -269,6 +269,8 @@
 )
 
 K8S_API_VERIFY_TLS = bool(strtobool(os.environ.get('K8S_API_VERIFY_TLS', 'true')))
+# drycc prometheus url
+DRYCC_PROMETHEUS_URL = os.environ.get('DRYCC_PROMETHEUS_URL', '')
 
 # security keys and auth tokens
 random_secret = 'CHANGEME_sapm$s%upvsw5l_zuy_&29rkywd^78ff(qi*#@&*^'