chore(monitor): use prometheus replace timeseries

Author: duanhongyi

charts/controller/templates/_helpers.tpl

@@ -90,26 +90,6 @@ env:
 - name: DRYCC_DATABASE_REPLICA_URL
   value: "postgres://$(DRYCC_PG_USER):$(DRYCC_PG_PASSWORD)@drycc-database-replica.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:5432/controller"
 {{- end }}
-{{- if (.Values.databaseMonitorUrl) }}
-- name: DRYCC_DATABASE_MONITOR_URL
-  valueFrom:
-    secretKeyRef:
-      name: controller-creds
-      key: database-monitor-url
-{{- else if eq .Values.global.timeseriesLocation "on-cluster"  }}
-- name: DRYCC_TS_USER
-  valueFrom:
-    secretKeyRef:
-      name: timeseries-creds
-      key: user
-- name: DRYCC_TS_PASSWORD
-  valueFrom:
-    secretKeyRef:
-      name: timeseries-creds
-      key: password
-- name: DRYCC_DATABASE_MONITOR_URL
-  value: "postgres://$(DRYCC_TS_USER):$(DRYCC_TS_PASSWORD)@drycc-timeseries-replica.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:5432/monitor"
-{{- end }}
 {{- if (.Values.workflowManagerUrl) }}
 - name: WORKFLOW_MANAGER_URL
   value: "{{ .Values.workflowManagerUrl }}"

charts/controller/templates/controller-secret-creds.yaml

@@ -11,9 +11,6 @@ data:
   {{- if (.Values.databaseReplicaUrl) }}
   database-replica-url: {{ .Values.databaseReplicaUrl | b64enc }}
   {{- end }}
-  {{- if (.Values.databaseMonitorUrl) }}
-  database-monitor-url: {{ .Values.databaseMonitorUrl | b64enc }}
-  {{- end }}
   {{- if (.Values.rabbitmqUrl) }}
   rabbitmq-url: {{ .Values.rabbitmqUrl | b64enc }}
   {{- end }}

charts/controller/values.yaml

@@ -84,7 +84,7 @@ service:
 # can be specified as key-value pairs under environment
 # this is usually a non required setting.
 environment:
-  RESERVED_NAMES: "drycc, drycc-builder, drycc-monitor-grafana, drycc-passport, drycc-helmbroker, drycc-manager"
+  RESERVED_NAMES: "drycc, drycc-builder, drycc-grafana, drycc-passport, drycc-helmbroker, drycc-manager"
 
 api:
   nodeAffinityPreset:

rootfs/api/management/commands/measure_loadbalancers.py

@@ -2,6 +2,7 @@
 import time
 import logging
 import ipaddress
+from asgiref.sync import async_to_sync
 from django.utils import timezone
 from django.core.management.base import BaseCommand
 from django.conf import settings
@@ -24,22 +25,23 @@ def _measure_loadbalancers(self, app_map, timestamp):
         stop = timestamp - (timestamp % 3600)
         start = stop - 3600
         loadbalancers = []
-        for item in monitor.query_loadbalancer(app_map.keys(), start, stop):
-            ip = item["ip"]
-            namespace = item["namespace"]
+        for metric, (_, value) in async_to_sync(monitor.query_loadbalancer(
+                app_map.keys(), start, stop)):
+            ip = metric["ip"]
+            namespace = metric["namespace"]
             owner_id = app_map[namespace].owner_id
             loadbalancers.append({
                 "app_id":  str(app_map[namespace].uuid),
                 "owner": owner_id,
                 "name": self._get_measure_name(ip),
                 "type": "loadbalancer",
                 "unit": "number",
-                "usage": 1,
+                "usage": value,
                 "kwargs": {
                     "ip": ip,
-                    "node": item["node"],
-                    "service": item["service"],
-                    "instance": item["instance"],
+                    "node": metric["node"],
+                    "service": metric["service"],
+                    "instance": metric["instance"],
                 },
                 "timestamp": start
             })

rootfs/api/management/commands/measure_networks.py

@@ -1,6 +1,7 @@
 import uuid
 import time
 import logging
+from asgiref.sync import async_to_sync
 from django.utils import timezone
 from django.core.management.base import BaseCommand
 from django.conf import settings
@@ -18,21 +19,34 @@ def _measure_networks(self, app_map, timestamp):
         stop = timestamp - (timestamp % 3600)
         start = stop - 3600
         networks = []
-        for namespace, pod_name, rx_bytes, tx_bytes in monitor.query_network_flow(
-                app_map.keys(), start, stop):
-            owner_id = app_map[namespace].owner_id
-            base_measure = {
-                "app_id":  str(app_map[namespace].uuid),
-                "owner": owner_id,
+        for metric, (_, value) in async_to_sync(monitor.query_network_receive_flow(
+                app_map.keys(), start, stop)):
+            networks.append({
+                "app_id":  str(app_map[metric['namespace']].uuid),
+                "owner": app_map[metric['namespace']].owner_id,
                 "type": "network",
                 "unit": "bytes",
+                "name": "rx",
+                "usage": value,
                 "kwargs": {
-                    "pod": pod_name,
+                    "pod": metric['pod'],
                 },
                 "timestamp": start
-            }
-            networks.append(base_measure | {"name": "rx", "usage": rx_bytes})
-            networks.append(base_measure | {"name": "tx", "usage": tx_bytes})
+            })
+        for metric, (_, value) in async_to_sync(monitor.query_network_transmit_flow(
+                app_map.keys(), start, stop)):
+            networks.append({
+                "app_id":  str(app_map[metric['namespace']].uuid),
+                "owner": app_map[metric['namespace']].owner_id,
+                "type": "network",
+                "unit": "bytes",
+                "name": "tx",
+                "usage": value,
+                "kwargs": {
+                    "pod": metric['pod'],
+                },
+                "timestamp": start
+            })
         send_measurements.delay(networks)
 
     def handle(self, *args, **options):

rootfs/api/models/config.py

@@ -174,7 +174,7 @@ def _update_values(self, previous_config):
                     else:  # force to string
                         new_item['value'] = str(new_item['value'])
                     break
-            if added:
+            if added and new_item['value'] is not None:
                 data.append(new_item)
         setattr(self, 'values', data)