chore(grafana): remove panel when component off-cluster

jianxiaoguo · jianxiaoguo · commit 176564f133e5 · 2022-11-09T15:15:29.000+08:00
diff --git a/charts/monitor/templates/_helpers.tmpl b/charts/monitor/templates/_helpers.tmpl
@@ -1,6 +1,10 @@
 {{/* Generate monitor grafana envs */}}
 {{- define "grafana.envs" }}
 env:
+- name: NAMESPACE
+  valueFrom:
+    fieldRef:
+      fieldPath: metadata.namespace
 {{- if eq .Values.global.influxdbLocation "off-cluster" }}
 - name: "INFLUXDB_URL"
   valueFrom:
@@ -73,4 +77,18 @@ env:
 - name: DRYCC_PASSPORT_SECRET
   value: "{{ .Values.grafana.passportSecret }}"
 {{- end }}
+- name: "INFLUXDB_LOCATION"
+  value: {{ .Values.global.influxdbLocation }}
+- name: "REDIS_LOCATION"
+  value: {{ .Values.global.redisLocation }}
+- name: "RABBITMQ_LOCATION"
+  value: {{ .Values.global.rabbitmqLocation }}
+- name: "STORAGE_LOCATION"
+  value: {{ .Values.global.storageLocation }}
+- name: "DATABASE_LOCATION"
+  value: {{ .Values.global.databaseLocation }}
+- name: "PASSPORT_LOCATION"
+  value: {{ .Values.global.passportLocation }}
+- name: "REGISTRY_LOCATION"
+  value: {{ .Values.global.registryLocation }}
 {{- end }}
diff --git a/charts/monitor/values.yaml b/charts/monitor/values.yaml
@@ -72,25 +72,19 @@ redis:
   port: "6379"
 
 global:
-  # Set the location of Workflow's influxdb cluster
+  # Set the location of Workflow's component cluster
   #
   # Valid values are:
-  # - on-cluster: Run Influxdb within the Kubernetes cluster
-  # - off-cluster: Influxdb is running outside of the cluster and credentials and connection information will be provided.
+  # - on-cluster: Run component within the Kubernetes cluster
+  # - off-cluster: component is running outside of the cluster and credentials and connection information will be provided.
   influxdbLocation: "on-cluster"
-  # Set the location of Workflow's grafana instance
-  #
-  # Valid values are:
-  # - on-cluster: Run Grafana within the Kubernetes cluster
-  # - off-cluster: Grafana is running outside of the cluster
   grafanaLocation: "on-cluster"
-  # Set the location of Workflow's redis instance
-  #
-  # Valid values are:
-  # - on-cluster: Run Redis within the Kubernetes cluster
-  # - off-cluster: Run Redis outside the Kubernetes cluster (configure in redis section)
   redisLocation: "on-cluster"
+  rabbitmqLocation: "on-cluster"
   storageLocation: "on-cluster"
+  databaseLocation: "on-cluster"
+  passportLocation: "on-cluster"
+  registryLocation: "on-cluster"
   # Role-Based Access Control for Kubernetes >= 1.5
   rbac: false
   ingressClass: ""
diff --git a/grafana/rootfs/Dockerfile b/grafana/rootfs/Dockerfile
@@ -4,7 +4,8 @@ ENV DRYCC_UID=1001 \
   DRYCC_GID=1001 \
   DRYCC_HOME_DIR=/usr/share/grafana \
   GRAFANA_VERSION="9.1.1" \
-  ENVTPL_VERSION="1.0.0"
+  ENVTPL_VERSION="1.0.0" \
+  JQ_VERSION="1.6"
 
 RUN groupadd drycc --gid ${DRYCC_GID} \
   && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR}
@@ -14,6 +15,7 @@ COPY . /
 RUN install-packages fontconfig curl \
   && install-stack grafana ${GRAFANA_VERSION} \
   && install-stack envtpl ${ENVTPL_VERSION} \
+  && install-stack jq ${JQ_VERSION} \
   && rm -rf \
       /usr/share/doc \
       /usr/share/man \
diff --git a/grafana/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json b/grafana/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json
diff --git a/grafana/rootfs/usr/share/grafana/api/dashboards/pd.json b/grafana/rootfs/usr/share/grafana/api/dashboards/pd.json
@@ -25,7 +25,7 @@
     "editable": true,
     "fiscalYearStartMonth": 0,
     "graphTooltip": 0,
-    "id": 5,
+    "id": null,
     "links": [],
     "liveNow": false,
     "panels": [
@@ -1059,10 +1059,7 @@
         },
         "targets": [
           {
-            "datasource": {
-              "type": "influxdb",
-              "uid": "6fcv2jV4z"
-            },
+            "datasource": null,
             "query": "from(bucket: \"kubernetes\")\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r[\"_measurement\"] == \"process_cpu_seconds_total\")\n  |> filter(fn: (r) => r[\"url\"] =~ /pd/)\n  |> toInt()\n  |> map(fn: (r) => ({ r with _value: r._value / 1000 }))\n  |> keep(columns: [\"_time\", \"_value\", \"url\"])\n  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n  |> yield(name: \"mean\")\n  ",
             "refId": "A"
           }
@@ -1162,11 +1159,6 @@
     "templating": {
       "list": [
         {
-          "current": {
-            "selected": false,
-            "text": "http://drycc-storage-meta-pd-0.drycc-storage-meta-pd.drycc.svc.cluster.local:2379/metrics",
-            "value": "http://drycc-storage-meta-pd-0.drycc-storage-meta-pd.drycc.svc.cluster.local:2379/metrics"
-          },
           "datasource": null,
           "definition": "from(bucket: v.bucket)\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r[\"_measurement\"] == \"pd_cluster_status\")\n  |> duplicate(column: \"url\", as: \"_value\")\n  |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\n  |> yield(name: \"last\")",
           "description": "pd_cluster_status url",
diff --git a/grafana/rootfs/usr/share/grafana/api/dashboards/redis.json b/grafana/rootfs/usr/share/grafana/api/dashboards/redis.json
@@ -100,7 +100,7 @@
             "measurement": "kubernetes_pod_container",
             "orderByTime": "ASC",
             "policy": "default",
-            "query": "from(bucket: v.bucket)\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r[\"_field\"] == \"cpu_usage_nanocores\")\n  |> filter(fn: (r) => r[\"namespace\"] == \"drycc\")\n  |> filter(fn: (r) => r[\"container_name\"] == \"drycc-redis\")\n  |> keep(columns: [\"_time\", \"node_name\", \"pod_name\", \"container_name\", \"_value\"])\n  |> movingAverage(n: 10)\n  |> map(fn: (r) => ({\n      r with\n      _value: r._value / 1000000.0\n    })\n  )\n  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n  |> yield(name: \"mean\")",
+            "query": "from(bucket: v.bucket)\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r[\"_field\"] == \"cpu_usage_nanocores\")\n  |> filter(fn: (r) => r[\"namespace\"] == \"drycc\")\n  |> filter(fn: (r) => r[\"container_name\"] == \"drycc-redis\")\n  |> keep(columns: [\"_time\", \"node_name\", \"namespace\", \"pod_name\", \"container_name\", \"_value\"])\n  |> movingAverage(n: 10)\n  |> map(fn: (r) => ({\n      r with\n      _value: r._value / 1000000.0\n    })\n  )\n  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n  |> yield(name: \"mean\")",
             "refId": "A",
             "resultFormat": "time_series"
           }
@@ -209,7 +209,7 @@
             "measurement": "kubernetes_pod_container",
             "orderByTime": "ASC",
             "policy": "default",
-            "query": "from(bucket: v.bucket)\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r[\"container_name\"] == \"drycc-redis\")\n  |> filter(fn: (r) => r[\"_field\"] == \"memory_usage_bytes\")\n  |> keep(columns: [\"_time\", \"node_name\", \"pod_name\", \"container_name\", \"_value\"])\n  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n  |> yield(name: \"mean\")",
+            "query": "from(bucket: v.bucket)\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"namespace\"] == \"drycc\")\n |> filter(fn: (r) => r[\"container_name\"] == \"drycc-redis\")\n  |> filter(fn: (r) => r[\"_field\"] == \"memory_usage_bytes\")\n  |> keep(columns: [\"_time\", \"node_name\", \"namespace\", \"pod_name\", \"container_name\", \"_value\"])\n  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n  |> yield(name: \"mean\")",
             "refId": "A",
             "resultFormat": "time_series"
           }
diff --git a/grafana/rootfs/usr/share/grafana/api/dashboards/tikv.json b/grafana/rootfs/usr/share/grafana/api/dashboards/tikv.json
@@ -25,7 +25,7 @@
     "editable": true,
     "fiscalYearStartMonth": 0,
     "graphTooltip": 0,
-    "id": 6,
+    "id": null,
     "links": [],
     "liveNow": false,
     "panels": [
@@ -881,7 +881,7 @@
         "gridPos": {
           "h": 7,
           "w": 12,
-          "x": 0,
+          "x": 12,
           "y": 15
         },
         "id": 2781,
@@ -947,132 +947,6 @@
         ],
         "title": "Memory",
         "type": "timeseries"
-      },
-      {
-        "datasource": null,
-        "description": "",
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "palette-classic"
-            },
-            "custom": {
-              "axisCenteredZero": false,
-              "axisColorMode": "text",
-              "axisLabel": "",
-              "axisPlacement": "auto",
-              "barAlignment": 0,
-              "drawStyle": "line",
-              "fillOpacity": 0,
-              "gradientMode": "none",
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              },
-              "lineInterpolation": "linear",
-              "lineWidth": 1,
-              "pointSize": 5,
-              "scaleDistribution": {
-                "type": "linear"
-              },
-              "showPoints": "auto",
-              "spanNulls": false,
-              "stacking": {
-                "group": "A",
-                "mode": "none"
-              },
-              "thresholdsStyle": {
-                "mode": "off"
-              }
-            },
-            "mappings": [],
-            "thresholds": {
-              "mode": "absolute",
-              "steps": [
-                {
-                  "color": "green",
-                  "value": null
-                },
-                {
-                  "color": "red",
-                  "value": 80
-                }
-              ]
-            },
-            "unit": "milli-cores"
-          },
-          "overrides": []
-        },
-        "gridPos": {
-          "h": 7,
-          "w": 12,
-          "x": 12,
-          "y": 15
-        },
-        "id": 2788,
-        "links": [],
-        "options": {
-          "legend": {
-            "calcs": [],
-            "displayMode": "list",
-            "placement": "right",
-            "showLegend": true
-          },
-          "tooltip": {
-            "mode": "multi",
-            "sort": "none"
-          }
-        },
-        "pluginVersion": "9.1.1",
-        "targets": [
-          {
-            "datasource": null,
-            "editorMode": "code",
-            "format": "time_series",
-            "intervalFactor": 2,
-            "legendFormat": "scheduler-{{instance}}",
-            "metric": "",
-            "query": "from(bucket: v.bucket)\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r[\"_measurement\"] == \"process_cpu_seconds_total\")\n  |> filter(fn: (r) => r[\"url\"] =~ /tikv/)\n  |> toInt()\n  |> map(fn: (r) => ({ r with _value: r._value / 1000 }))\n  |> keep(columns: [\"_time\", \"_value\", \"url\"])\n  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n  |> yield(name: \"mean\")\n  ",
-            "range": true,
-            "refId": "A",
-            "step": 4
-          },
-          {
-            "datasource": null,
-            "editorMode": "code",
-            "format": "time_series",
-            "intervalFactor": 2,
-            "legendFormat": "channelfull-{{instance}}-{{type}}",
-            "metric": "",
-            "query": "",
-            "range": true,
-            "refId": "B",
-            "step": 4
-          },
-          {
-            "datasource": null,
-            "editorMode": "code",
-            "format": "time_series",
-            "intervalFactor": 2,
-            "legendFormat": "coprocessor-{{instance}}",
-            "metric": "",
-            "range": true,
-            "refId": "C",
-            "step": 4
-          },
-          {
-            "datasource": null,
-            "editorMode": "code",
-            "format": "time_series",
-            "intervalFactor": 2,
-            "legendFormat": "stall-{{instance}}",
-            "range": true,
-            "refId": "D"
-          }
-        ],
-        "title": "CPU",
-        "type": "timeseries"
       }
     ],
     "schemaVersion": 37,
@@ -1082,11 +956,6 @@
       "list": [
         {
           "allValue": "",
-          "current": {
-            "selected": false,
-            "text": "http://drycc-storage-meta-tikv-0.drycc-storage-meta-tikv.drycc.svc.cluster.local:20180/metrics",
-            "value": "http://drycc-storage-meta-tikv-0.drycc-storage-meta-tikv.drycc.svc.cluster.local:20180/metrics"
-          },
           "datasource": null,
           "definition": "from(bucket: v.bucket)\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r[\"_measurement\"] == \"tikv_engine_size_bytes\")\n  |> filter(fn: (r) => r[\"_field\"] == \"gauge\")\n  |> keyValues(keyColumns: [\"_value\"])\n  |> duplicate(column: \"url\", as: \"_value\")\n  |> group() \n  |> keep(columns: [\"_value\"])\n  ",
           "hide": 0,
diff --git a/grafana/rootfs/usr/share/grafana/start-grafana b/grafana/rootfs/usr/share/grafana/start-grafana
@@ -4,6 +4,7 @@
 HEADER_CONTENT_TYPE="Content-Type: application/json"
 HEADER_ACCEPT="Accept: application/json"
 
+NAMESPACE=${NAMESPACE:-drycc}
 GRAFANA_USER=${DEFAULT_USER:-admin}
 GRAFANA_PASSWD=${DEFAULT_USER_PASSWORD:-admin}
 GRAFANA_PORT=${BIND_PORT:-3000}
@@ -102,6 +103,52 @@ curl -i -XPOST -H "${HEADER_ACCEPT}" -H "${HEADER_CONTENT_TYPE}" "http://${GRAFA
 
 echo ""
 echo "Importing default dashboards..."
+# replace namespace
+# shellcheck disable=SC1003
+# shellcheck disable=SC2046
+sed -i 's@r\[\\"namespace\\\"\] == \\"drycc\\"@r\[\\"namespace\\"\] == \\"\'"${NAMESPACE}"'\\"@g' $(grep -F 'r[\"namespace\"] == \"drycc\"' -rl --include="*.json" "${DASHBOARD_LOCATION}")
+# drycc component dashboard
+DCD="${DASHBOARD_LOCATION}"/drycc_component_health.json
+
+# remove off-cluster component panel
+if [ "on-cluster" != "${INFLUXDB_LOCATION}" ]; then
+  rm -rf "${DASHBOARD_LOCATION}"/influx.json
+  # shellcheck disable=SC2005
+  echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "INFLUXDB"))')" > "${DCD}"
+fi
+if [ "on-cluster" != "${REDIS_LOCATION}" ]; then
+  rm -rf "${DASHBOARD_LOCATION}"/redis.json
+  # shellcheck disable=SC2005
+  echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "REDIS"))')" > "${DCD}"
+fi
+if [ "on-cluster" != "${RABBITMQ_LOCATION}" ]; then
+  # shellcheck disable=SC2005
+  echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "RABBITMQ"))')" > "${DCD}"
+fi
+if [ "on-cluster" != "${DATABASE_LOCATION}" ]; then
+  # shellcheck disable=SC2005
+  echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "DATABASE"))')" > "${DCD}"
+fi
+if [ "on-cluster" != "${PASSPORT_LOCATION}" ]; then
+  # shellcheck disable=SC2005
+  echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "PASSPORT"))')" > "${DCD}"
+fi
+if [ "on-cluster" != "${REGISTRY_LOCATION}" ]; then
+  # shellcheck disable=SC2005
+  echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "REGISTRY"))')" > "${DCD}"
+  # shellcheck disable=SC2005
+  echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "REGISTRY-PROXY"))')" > "${DCD}"
+fi
+if [ "on-cluster" != "${STORAGE_LOCATION}" ]; then
+  rm -rf "${DASHBOARD_LOCATION}"/pd.json "${DASHBOARD_LOCATION}"/tikv.json
+  # shellcheck disable=SC2005
+  echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "STORAGE-MINIO"))')" > "${DCD}"
+  # shellcheck disable=SC2005
+  echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "STORAGE-META-PD"))')" > "${DCD}"
+  # shellcheck disable=SC2005
+  echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "STORAGE-META-TIKV"))')" > "${DCD}"
+fi
+
 for filename in "${DASHBOARD_LOCATION}"/*.json; do
   echo "Importing ${filename} ..."
   curl -i -XPOST --data "@${filename}" -H "${HEADER_ACCEPT}" -H "${HEADER_CONTENT_TYPE}" "http://${GRAFANA_USER}:${GRAFANA_PASSWD}@localhost:${GRAFANA_PORT}/api/dashboards/db"
