From 62e9f8eb5aef20aba6841910f8cd7a38c8c8227d Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Thu, 10 Oct 2024 09:43:49 +0800 Subject: [PATCH 01/50] chore(charts): remove k8s early compatibility --- charts/monitor/templates/monitor-telegraf-daemon.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/monitor/templates/monitor-telegraf-daemon.yaml b/charts/monitor/templates/monitor-telegraf-daemon.yaml index 9654110..6f679a0 100644 --- a/charts/monitor/templates/monitor-telegraf-daemon.yaml +++ b/charts/monitor/templates/monitor-telegraf-daemon.yaml @@ -10,10 +10,8 @@ spec: selector: matchLabels: app: drycc-monitor-telegraf - {{- if ge .Capabilities.KubeVersion.Minor "6" }} updateStrategy: type: RollingUpdate - {{- end }} template: metadata: labels: {{- include "common.labels.standard" . | nindent 8 }} From 163c7f6b6a89bb5286a4f0587f95163bd5fb8f24 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 30 Oct 2024 17:40:25 +0800 Subject: [PATCH 02/50] chore(monitor): bump grafana to 11 and telegraf to 1.32 --- grafana/rootfs/Dockerfile | 4 ++-- telegraf/rootfs/Dockerfile | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/grafana/rootfs/Dockerfile b/grafana/rootfs/Dockerfile index 53354e6..e7dd282 100644 --- a/grafana/rootfs/Dockerfile +++ b/grafana/rootfs/Dockerfile @@ -4,9 +4,9 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/usr/share/grafana \ - GRAFANA_VERSION="9.5.3" \ + GRAFANA_VERSION="11.1.3" \ ENVTPL_VERSION="1.0.0" \ - JQ_VERSION="1.6" + JQ_VERSION="1.7.1" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} diff --git a/telegraf/rootfs/Dockerfile b/telegraf/rootfs/Dockerfile index a49a2b6..0c743f7 100644 --- a/telegraf/rootfs/Dockerfile +++ b/telegraf/rootfs/Dockerfile @@ -4,10 +4,10 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/home/telegraf \ - MC_VERSION="2023.06.15.15.08.26" \ - TELEGRAF_VERSION="1.30.2" \ + MC_VERSION="2024.10.08.09.37.26" \ + TELEGRAF_VERSION="1.32.2" \ ENVTPL_VERSION="1.0.0" \ - JQ_VERSION="1.6" + JQ_VERSION="1.7.1" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} From c04f940b74957d23523313f43161d700247aa8fc Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Thu, 31 Oct 2024 08:58:48 +0800 Subject: [PATCH 03/50] fix(fluentbit): fluentbit cpu error --- README.md | 20 +++++++++---------- .../dashboards/drycc_component_health.json | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 9ca60bc..5f61421 100644 --- a/README.md +++ b/README.md @@ -26,24 +26,24 @@ Grafana is a stand alone graphing application. It natively supports Timescaledb │ │ logs file │ │ │ - ▼ ▼ -┌──────────┐ ┌───-─────────┐ logs/metrics ┌──────────────┐ -│ App Logs │───Log File───▶│ Fluentd │────topics──────▶│ Redis Stream │ -└──────────┘ └────-────────┘ └──────────────┘ + v v +┌──────────┐ ┌─────────────┐ logs/metrics ┌──────────────┐ +│ App Logs │───Log File──> │ Fluentbit │──────topics────>│ Redis Stream │ +└──────────┘ └─────────────┘ └──────────────┘ │ ┌──────────┐ │ │ HOST │───┐ │ │ Telegraf │ │ │ └──────────┘ │ │ - │ ▼ -┌──────────┐ │ ┌───────-─────┐ ┌─────────────┐ -│ HOST │───┼──────────▶│ Timescaledb │◀──────Wire──────│ Telegraf │ -│ Telegraf │ │ └───────────-─┘ Protocol └─────────────┘ + │ v +┌──────────┐ │ ┌─────────────┐ ┌─────────────┐ +│ HOST │───┼──────────>│ Timescaledb │<──────Wire──────│ Telegraf │ +│ Telegraf │ │ └─────────────┘ Protocol └─────────────┘ └──────────┘ │ │ │ │ -┌──────────┐ │ ▼ +┌──────────┐ │ v │ HOST │───┘ ┌─────────────┐ ┌─────────────┐ -│ Telegraf │ │ Grafana │◀────────────────│ Prometheus │ +│ Telegraf │ │ Grafana │<────────────────│ Prometheus │ └──────────┘ └─────────────┘ └─────────────┘ ``` diff --git a/grafana/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json b/grafana/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json index 86bce04..4faee35 100644 --- a/grafana/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json +++ b/grafana/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json @@ -1277,7 +1277,7 @@ { "alias": "$tag_pod_name", "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-logger-fluentd.*$\"}[1m])) by (pod) * 1000", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-logger-fluentbit.*$\"}[1m])) by (pod) * 1000", "groupBy": [ { "params": [ @@ -1386,7 +1386,7 @@ { "alias": "$tag_pod_name", "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-logger-fluent.*$\"}) by (pod)", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-logger-fluentbit.*$\"}) by (pod)", "groupBy": [ { "params": [ @@ -1452,7 +1452,7 @@ "refId": "A" } ], - "title": "FLUENTD", + "title": "FLUENTBIT", "type": "row" }, { From 57b7adfd6de7815e8f26467c936b4badd82697b2 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 13 Nov 2024 18:06:43 +0800 Subject: [PATCH 04/50] chore(grafana): add rabbitmq dashboard --- .../api/dashboards/drycc_rabbitmq.json | 8212 +++++++++++++++++ 1 file changed, 8212 insertions(+) create mode 100644 grafana/rootfs/usr/share/grafana/api/dashboards/drycc_rabbitmq.json diff --git a/grafana/rootfs/usr/share/grafana/api/dashboards/drycc_rabbitmq.json b/grafana/rootfs/usr/share/grafana/api/dashboards/drycc_rabbitmq.json new file mode 100644 index 0000000..41d15e7 --- /dev/null +++ b/grafana/rootfs/usr/share/grafana/api/dashboards/drycc_rabbitmq.json @@ -0,0 +1,8212 @@ +{ + "dashboard": { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "8.3.4" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "A new RabbitMQ Management Overview", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "iteration": 1659711638455, + "links": [ + { + "icon": "doc", + "tags": [], + "targetBlank": true, + "title": "Monitoring with Prometheus & Grafana", + "tooltip": "", + "type": "link", + "url": "https://www.rabbitmq.com/prometheus.html" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#37872D", + "value": null + }, + { + "color": "#1F60C4", + "value": 10000 + }, + { + "color": "#C4162A", + "value": 100000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "id": 64, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_ready * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Ready messages", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": -1 + }, + { + "color": "#37872D", + "value": 50 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 62, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_received_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Incoming messages / s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 66, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_global_publishers * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Publishers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 37, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_connections * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 40, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_queues * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Queues", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#37872D", + "value": null + }, + { + "color": "#1F60C4", + "value": 100 + }, + { + "color": "#C4162A", + "value": 500 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 65, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_unacked * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Unacknowledged messages", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": -1 + }, + { + "color": "#37872D", + "value": 50 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 63, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_redelivered_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_global_messages_delivered_consume_auto_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_global_messages_delivered_consume_manual_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_global_messages_delivered_get_auto_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_global_messages_delivered_get_manual_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Outgoing messages / s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 3 + }, + "id": 41, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_consumers * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Consumers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 3 + }, + "id": 38, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_channels * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Channels", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1F60C4", + "value": null + }, + { + "color": "#37872D", + "value": 3 + }, + { + "color": "#C4162A", + "value": 8 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 3 + }, + "id": 67, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_build_info * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Nodes", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 4, + "panels": [], + "title": "NODES", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "erlang_version" + }, + "properties": [ + { + "id": "displayName", + "value": "Erlang/OTP" + }, + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)" + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "rabbitmq_version" + }, + "properties": [ + { + "id": "displayName", + "value": "RabbitMQ" + }, + { + "id": "unit", + "value": "none" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)" + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "displayName", + "value": "Host" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "rabbitmq_node" + }, + "properties": [ + { + "id": "displayName", + "value": "Node name" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)" + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "rabbitmq_cluster" + }, + "properties": [ + { + "id": "displayName", + "value": "Cluster" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "prometheus_client_version" + }, + "properties": [ + { + "id": "displayName", + "value": "prometheus.erl" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "prometheus_plugin_version" + }, + "properties": [ + { + "id": "displayName", + "value": "rabbitmq_prometheus" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 69, + "links": [], + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "rabbitmq_build_info * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "table", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "If the value is zero or less, the memory alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the memory alarm is triggered with a slight delay.\n\nThe kernel's view of the amount of memory used by the node can differ from what the node itself can observe. This means that this value can be negative for a sustained period of time.\n\nBy default nodes use resident set size (RSS) to compute how much memory they use. This strategy can be changed (see the guides below).\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Memory Alarms](https://www.rabbitmq.com/memory.html)\n* [Reasoning About Memory Use](https://www.rabbitmq.com/memory-use.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "decimals": 1, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0 + }, + { + "color": "transparent", + "value": 536870912 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 7, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "(rabbitmq_resident_memory_limit_bytes * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) -\n(rabbitmq_process_resident_memory_bytes * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Memory available before publishers blocked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric is reported for the partition where the RabbitMQ data directory is stored.\n\nIf the value is zero or less, the disk alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the free disk space alarm is triggered with a slight delay.\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Disk Space Alarms](https://www.rabbitmq.com/disk-alarms.html)\n* [Disk Space](https://www.rabbitmq.com/production-checklist.html#resource-limits-disk-space)\n* [Persistence Configuration](https://www.rabbitmq.com/persistence-conf.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "decimals": 1, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 1073741824 + }, + { + "color": "transparent", + "value": 5368709120 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 12, + "y": 11 + }, + "id": 8, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "rabbitmq_disk_space_available_bytes * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Disk space available before publishers blocked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "When this value reaches zero, new connections will not be accepted and disk write operations may fail.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Open File Handles Limit](https://www.rabbitmq.com/production-checklist.html#resource-limits-file-handle-limit)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 500 + }, + { + "color": "transparent", + "value": 1000 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 11 + }, + "id": 2, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "(rabbitmq_process_max_fds * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) -\n(rabbitmq_process_open_fds * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "File descriptors available", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "When this value reaches zero, new connections will not be accepted.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Networking and RabbitMQ](https://www.rabbitmq.com/networking.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 500 + }, + { + "color": "transparent", + "value": 1000 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 15 + }, + "id": 5, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "(rabbitmq_process_max_tcp_sockets * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) -\n(rabbitmq_process_open_tcp_sockets * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "TCP sockets available", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 27, + "panels": [], + "title": "QUEUED MESSAGES", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of ready messages ready to be delivered to consumers.\n\nAim to keep this value as low as possible. RabbitMQ behaves best when messages are flowing through it. It's OK for publishers to occasionally outpace consumers, but the expectation is that consumers will eventually process all ready messages.\n\nIf this metric keeps increasing, your system will eventually run out of memory and/or disk space. Consider using TTL or Queue Length Limit to prevent unbounded message growth.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Consumers](https://www.rabbitmq.com/consumers.html)\n* [Queue Length Limit](https://www.rabbitmq.com/maxlength.html)\n* [Time-To-Live and Expiration](https://www.rabbitmq.com/ttl.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 9, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_ready * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages ready to be delivered to consumers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The total number of messages that are either in-flight to consumers, currently being processed by consumers or simply waiting for the consumer acknowledgements to be processed by the queue. Until the queue processes the message acknowledgement, the message will remain unacknowledged.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 19, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_unacked * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages pending consumer acknowledgement", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 11, + "panels": [], + "title": "INCOMING MESSAGES", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The incoming message rate before any routing rules are applied.\n\nIf this value is lower than the number of messages published to queues, it may indicate that some messages are delivered to more than one queue.\n\nIf this value is higher than the number of messages published to queues, messages cannot be routed and will either be dropped or returned to publishers.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 13, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_received_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages published / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages confirmed by the broker to publishers. Publishers must opt-in to receive message confirmations.\n\nIf this metric is consistently at zero it may suggest that publisher confirms are not used by clients. The safety of published messages is likely to be at risk.\n\n* [Publisher Confirms](https://www.rabbitmq.com/confirms.html#publisher-confirms)\n* [Publisher Confirms and Data Safety](https://www.rabbitmq.com/publishers.html#data-safety)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 18, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_confirmed_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages confirmed to publishers / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages received from publishers and successfully routed to the master queue replicas.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Publishers](https://www.rabbitmq.com/publishers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 61, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_routed_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages routed to queues / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages received from publishers that have publisher confirms enabled and the broker has not confirmed yet.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 12, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_received_confirm_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"} - \nrate(rabbitmq_global_messages_confirmed_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}\n) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages unconfirmed to publishers / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages that cannot be routed and are dropped. \n\nAny value above zero means message loss and likely suggests a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/rabbit/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 34, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_unroutable_dropped_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Unroutable messages dropped / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages that cannot be routed and are returned back to publishers.\n\nSustained values above zero may indicate a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/rabbit/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 16, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_unroutable_returned_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Unroutable messages returned to publishers / s", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 29, + "panels": [], + "title": "OUTGOING MESSAGES", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages delivered to consumers. It includes messages that have been redelivered.\n\nThis metric does not include messages that have been fetched by consumers using `basic.get` (consumed by polling).\n\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 14, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(\n (rate(rabbitmq_global_messages_delivered_consume_auto_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\n (rate(rabbitmq_global_messages_delivered_consume_manual_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})\n) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages delivered / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages that have been redelivered to consumers. It includes messages that have been requeued automatically and redelivered due to channel exceptions or connection closures.\n\nHaving some redeliveries is expected, but if this metric is consistently non-zero, it is worth investigating why.\n\n* [Negative Acknowledgement and Requeuing of Deliveries](https://www.rabbitmq.com/confirms.html#consumer-nacks-requeue)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 20 + }, + { + "color": "red", + "value": 100 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 15, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_redelivered_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages redelivered / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of message deliveries to consumers that use manual acknowledgement mode.\n\nWhen this mode is used, RabbitMQ waits for consumers to acknowledge messages before more messages can be delivered.\n\nThis is the safest way of consuming messages.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 20, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_delivered_consume_manual_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages delivered with manual ack / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of message deliveries to consumers that use automatic acknowledgement mode.\n\nWhen this mode is used, RabbitMQ does not wait for consumers to acknowledge message deliveries.\n\nThis mode is fire-and-forget and does not offer any delivery safety guarantees. It tends to provide higher throughput and it may lead to consumer overload and higher consumer memory usage.\n\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 21, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_delivered_consume_auto_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages delivered auto ack / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of message acknowledgements coming from consumers that use manual acknowledgement mode.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 52 + }, + "id": 22, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_acknowledged_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages acknowledged / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages delivered to polling consumers that use automatic acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/rabbit/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 52 + }, + "id": 24, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_delivered_get_auto_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Polling operations with auto ack / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of polling consumer operations that yield no result.\n\nAny value above zero means that RabbitMQ resources are wasted by polling consumers.\n\nCompare this metric to the other polling consumer metrics to see the inefficiency rate.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/rabbit/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 25, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_get_empty_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Polling operations that yield no result / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages delivered to polling consumers that use manual acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/rabbit/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 23, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_delivered_get_manual_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Polling operations with manual ack / s", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 53, + "panels": [], + "title": "QUEUES", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of queue masters per node. \n\nThis metric makes it easy to see sub-optimal queue distribution in a cluster.\n\n* [Queue Masters, Data Locality](https://www.rabbitmq.com/ha.html#master-migration-data-locality)\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 57, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "rabbitmq_queues * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Total queues", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of queue declarations performed by clients.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 63 + }, + "id": 58, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_declared_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Queues declared / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of new queues created (as opposed to redeclarations).\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 63 + }, + "id": 60, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_created_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Queues created / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of queues deleted.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 63 + }, + "id": 59, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_deleted_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Queues deleted / s", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 51, + "panels": [], + "title": "CHANNELS", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of channels on all currently opened connections.\n\nIf this metric grows monotonically it is highly likely a channel leak in one of the applications. Confirm channel leaks by using the _Channels opened_ and _Channels closed_ metrics.\n\n* [Channel Leak](https://www.rabbitmq.com/channels.html#channel-leaks)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 54, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "rabbitmq_channels * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Total channels", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of new channels opened by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 69 + }, + "id": 55, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_channels_opened_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Channels opened / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of channels closed by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 69 + }, + "id": 56, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_channels_closed_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Channels closed / s", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 46, + "panels": [], + "title": "CONNECTIONS", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of client connections.\n\nIf this metric grows monotonically it is highly likely a connection leak in one of the applications. Confirm connection leaks by using the _Connections opened_ and _Connections closed_ metrics.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 47, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "rabbitmq_connections * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Total connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of new connections opened by clients. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 75 + }, + "id": 48, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_connections_opened_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Connections opened / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of connections closed. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 75 + }, + "id": 49, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_connections_closed_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Connections closed / s", + "type": "timeseries" + } + ], + "refresh": "15s", + "schemaVersion": 34, + "style": "dark", + "tags": [ + "rabbitmq-prometheus" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "datasource": "PBFA97CFB590B2093", + "hide": 2, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(rabbitmq_identity_info, namespace)", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(rabbitmq_identity_info, namespace)", + "refId": "Prometheus-namespace-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(rabbitmq_identity_info{namespace=\"$namespace\"}, rabbitmq_cluster)", + "hide": 0, + "includeAll": false, + "label": "RabbitMQ Cluster", + "multi": false, + "name": "rabbitmq_cluster", + "options": [], + "query": { + "query": "label_values(rabbitmq_identity_info{namespace=\"$namespace\"}, rabbitmq_cluster)", + "refId": "Prometheus-rabbitmq_cluster-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "15s", + "30s", + "1m", + "5m", + "10m" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "RabbitMQ-Overview", + "uid": "Kn5xm-gZk", + "version": 20220805, + "weekStart": "", + "gnetId": 10991 + }, + "overwrite": false +} \ No newline at end of file From b87f37079759ddeb182c2c0d69735ffbc05ada02 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Mon, 18 Nov 2024 16:21:43 +0800 Subject: [PATCH 05/50] chore(monitor): remove telegraf --- .gitignore | 5 - .woodpecker/build-linux.yml | 3 +- .woodpecker/chart.yaml | 3 +- .woodpecker/manifest-telegraf.yml | 38 --- .../{manifest-grafana.yml => manifest.yml} | 2 +- .woodpecker/test-linux.yml | 3 +- default.mk => Makefile | 17 +- README.md | 171 ++++++++++--- charts/grafana/Chart.lock | 6 + charts/{monitor => grafana}/Chart.yaml | 6 +- charts/grafana/charts/common-1.1.3.tgz | Bin 0 -> 10632 bytes .../templates/_helpers.tmpl | 50 +--- .../templates/grafana-deployment.yaml} | 30 +-- charts/grafana/templates/grafana-pvc.yaml | 24 ++ .../templates/grafana-svc.yaml} | 12 +- charts/{monitor => grafana}/values.yaml | 90 +++---- .../templates/monitor-clusterrole.yaml | 14 - .../templates/monitor-grafana-pvc.yaml | 24 -- .../monitor-telegraf-clusterrole.yaml | 17 -- .../monitor-telegraf-clusterrolebinding.yaml | 18 -- .../templates/monitor-telegraf-daemon.yaml | 53 ---- .../monitor-telegraf-service-account.yaml | 6 - deploy.mk | 9 - grafana/Makefile | 4 - grafana/README.md | 122 --------- {grafana/rootfs => rootfs}/Dockerfile | 0 .../dashboards/drycc_component_health.json | 241 +----------------- .../api/dashboards/drycc_rabbitmq.json | 0 .../grafana/api/dashboards/drycc_storage.json | 0 .../kubernetes_deployment_metrics.json | 0 .../share/grafana/api/dashboards/node.json | 0 .../usr/share/grafana/api/dashboards/pd.json | 0 .../grafana/api/dashboards/tikv_details.json | 0 .../grafana/api/dashboards/tikv_summary.json | 0 .../api/dashboards/tikv_trouble_shooting.json | 0 .../usr/share/grafana/entrypoint.sh | 0 .../usr/share/grafana/grafana.ini.tpl | 0 .../usr/share/grafana/ldap.toml.tpl | 0 .../usr/share/grafana/start-grafana | 0 telegraf/Makefile | 4 - telegraf/README.md | 43 ---- telegraf/rootfs/Dockerfile | 39 --- telegraf/rootfs/home/telegraf/config.toml.tpl | 114 --------- telegraf/rootfs/home/telegraf/start-telegraf | 30 --- 44 files changed, 239 insertions(+), 959 deletions(-) delete mode 100644 .woodpecker/manifest-telegraf.yml rename .woodpecker/{manifest-grafana.yml => manifest.yml} (90%) rename default.mk => Makefile (59%) create mode 100644 charts/grafana/Chart.lock rename charts/{monitor => grafana}/Chart.yaml (68%) create mode 100644 charts/grafana/charts/common-1.1.3.tgz rename charts/{monitor => grafana}/templates/_helpers.tmpl (62%) rename charts/{monitor/templates/monitor-grafana-deployment.yaml => grafana/templates/grafana-deployment.yaml} (60%) create mode 100644 charts/grafana/templates/grafana-pvc.yaml rename charts/{monitor/templates/monitor-grafana-svc.yaml => grafana/templates/grafana-svc.yaml} (57%) rename charts/{monitor => grafana}/values.yaml (58%) delete mode 100644 charts/monitor/templates/monitor-clusterrole.yaml delete mode 100644 charts/monitor/templates/monitor-grafana-pvc.yaml delete mode 100644 charts/monitor/templates/monitor-telegraf-clusterrole.yaml delete mode 100644 charts/monitor/templates/monitor-telegraf-clusterrolebinding.yaml delete mode 100644 charts/monitor/templates/monitor-telegraf-daemon.yaml delete mode 100644 charts/monitor/templates/monitor-telegraf-service-account.yaml delete mode 100644 deploy.mk delete mode 100644 grafana/Makefile delete mode 100644 grafana/README.md rename {grafana/rootfs => rootfs}/Dockerfile (100%) rename {grafana/rootfs => rootfs}/usr/share/grafana/api/dashboards/drycc_component_health.json (93%) rename {grafana/rootfs => rootfs}/usr/share/grafana/api/dashboards/drycc_rabbitmq.json (100%) rename {grafana/rootfs => rootfs}/usr/share/grafana/api/dashboards/drycc_storage.json (100%) rename {grafana/rootfs => rootfs}/usr/share/grafana/api/dashboards/kubernetes_deployment_metrics.json (100%) rename {grafana/rootfs => rootfs}/usr/share/grafana/api/dashboards/node.json (100%) rename {grafana/rootfs => rootfs}/usr/share/grafana/api/dashboards/pd.json (100%) rename {grafana/rootfs => rootfs}/usr/share/grafana/api/dashboards/tikv_details.json (100%) rename {grafana/rootfs => rootfs}/usr/share/grafana/api/dashboards/tikv_summary.json (100%) rename {grafana/rootfs => rootfs}/usr/share/grafana/api/dashboards/tikv_trouble_shooting.json (100%) rename {grafana/rootfs => rootfs}/usr/share/grafana/entrypoint.sh (100%) rename {grafana/rootfs => rootfs}/usr/share/grafana/grafana.ini.tpl (100%) rename {grafana/rootfs => rootfs}/usr/share/grafana/ldap.toml.tpl (100%) rename {grafana/rootfs => rootfs}/usr/share/grafana/start-grafana (100%) delete mode 100644 telegraf/Makefile delete mode 100644 telegraf/README.md delete mode 100644 telegraf/rootfs/Dockerfile delete mode 100644 telegraf/rootfs/home/telegraf/config.toml.tpl delete mode 100755 telegraf/rootfs/home/telegraf/start-telegraf diff --git a/.gitignore b/.gitignore index 72d9643..edd7126 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,3 @@ *.swp *.swo .DS_Store - -telegraf/manifests/*.tmp.yaml -chronograf/manifests/*.tmp.yaml -grafana/manifests/*.tmp.yaml -rootfs/usr/bin/telegraf diff --git a/.woodpecker/build-linux.yml b/.woodpecker/build-linux.yml index 0fe1f11..5ddefd6 100644 --- a/.woodpecker/build-linux.yml +++ b/.woodpecker/build-linux.yml @@ -13,8 +13,7 @@ steps: commands: - echo $CONTAINER_PASSWORD | podman login $DRYCC_REGISTRY --username $CONTAINER_USERNAME --password-stdin > /dev/null 2>&1 - export VERSION=$([ -z $CI_COMMIT_TAG ] && echo latest || echo $CI_COMMIT_TAG)-$(sed 's#/#-#g' <<< $CI_SYSTEM_PLATFORM) - - cd grafana && make build-all push-all && cd .. - - cd telegraf && make build-all push-all && cd .. + - make podman-build podman-push secrets: - codename - dev_registry diff --git a/.woodpecker/chart.yaml b/.woodpecker/chart.yaml index 21abe1e..5aa7eb3 100644 --- a/.woodpecker/chart.yaml +++ b/.woodpecker/chart.yaml @@ -25,5 +25,4 @@ steps: - tag depends_on: -- manifest-grafana -- manifest-telegraf +- manifest diff --git a/.woodpecker/manifest-telegraf.yml b/.woodpecker/manifest-telegraf.yml deleted file mode 100644 index 6351863..0000000 --- a/.woodpecker/manifest-telegraf.yml +++ /dev/null @@ -1,38 +0,0 @@ -labels: - type: exec - platform: linux/amd64 - -steps: -- name: generate-manifest - image: bash - commands: - - sed -i "s/{{project}}/telegraf/g" .woodpecker/manifest.tmpl - - sed -i "s/registry.drycc.cc/$${DRYCC_REGISTRY}/g" .woodpecker/manifest.tmpl - secrets: - - drycc_registry - when: - event: - - tag - - push - -- name: publish-manifest - image: bash - commands: - - podman run --rm - -e PLUGIN_SPEC=.woodpecker/manifest.tmpl - -e PLUGIN_USERNAME=$CONTAINER_USERNAME - -e PLUGIN_PASSWORD=$CONTAINER_PASSWORD - -e DRONE_TAG=$CI_COMMIT_TAG - -v $(pwd):$(pwd) - -w $(pwd) - plugins/manifest - secrets: - - container_username - - container_password - when: - event: - - tag - - push - -depends_on: -- build-linux diff --git a/.woodpecker/manifest-grafana.yml b/.woodpecker/manifest.yml similarity index 90% rename from .woodpecker/manifest-grafana.yml rename to .woodpecker/manifest.yml index 66bf341..052f5d8 100644 --- a/.woodpecker/manifest-grafana.yml +++ b/.woodpecker/manifest.yml @@ -6,7 +6,7 @@ steps: - name: generate-manifest image: bash commands: - - sed -i "s/{{project}}/grafana/g" .woodpecker/manifest.tmpl + - sed -i "s/{{project}}/$${CI_REPO_NAME}/g" .woodpecker/manifest.tmpl - sed -i "s/registry.drycc.cc/$${DRYCC_REGISTRY}/g" .woodpecker/manifest.tmpl secrets: - drycc_registry diff --git a/.woodpecker/test-linux.yml b/.woodpecker/test-linux.yml index 4e08a52..cbb0cc6 100644 --- a/.woodpecker/test-linux.yml +++ b/.woodpecker/test-linux.yml @@ -11,8 +11,7 @@ steps: - name: test-linux image: bash commands: - - cd grafana && make test && cd .. - - cd telegraf && make test && cd .. + - make test secrets: - codename - dev_registry diff --git a/default.mk b/Makefile similarity index 59% rename from default.mk rename to Makefile index 2af118b..b8b41c1 100644 --- a/default.mk +++ b/Makefile @@ -1,14 +1,16 @@ SHELL := /bin/bash -# all monitor components share/use the following targets/exports +# grafana share/use the following targets/exports +SHORT_NAME ?= grafana +SHELL_SCRIPTS = rootfs/usr/share/grafana/start-grafana + BUILD_TAG ?= git-$(shell git rev-parse --short HEAD) DRYCC_REGISTRY ?= ${DEV_REGISTRY} IMAGE_PREFIX ?= drycc PLATFORM ?= linux/amd64,linux/arm64 -include ../includes.mk -include ../versioning.mk -include ../deploy.mk +include includes.mk +include versioning.mk TEST_ENV_PREFIX := podman run --rm -v ${CURDIR}:/bash -w /bash ${DEV_REGISTRY}/drycc/go-dev @@ -33,10 +35,3 @@ test-style: .PHONY: build push podman-build clean upgrade deploy test test-style -build-all: - podman build --build-arg CODENAME=${CODENAME} -t ${DRYCC_REGISTRY}/${IMAGE_PREFIX}/grafana:${VERSION} ../grafana/rootfs - podman build --build-arg CODENAME=${CODENAME} -t ${DRYCC_REGISTRY}/${IMAGE_PREFIX}/telegraf:${VERSION} ../telegraf/rootfs - -push-all: - podman push ${DRYCC_REGISTRY}/${IMAGE_PREFIX}/grafana:${VERSION} - podman push ${DRYCC_REGISTRY}/${IMAGE_PREFIX}/telegraf:${VERSION} diff --git a/README.md b/README.md index 5f61421..3d2a264 100644 --- a/README.md +++ b/README.md @@ -1,54 +1,141 @@ -# Drycc Monitor v2 +# Drycc Grafana -[![Build Status](https://woodpecker.drycc.cc/api/badges/drycc/monitor/status.svg)](https://woodpecker.drycc.cc/drycc/monitor) +[![Build Status](https://woodpecker.drycc.cc/api/badges/drycc/grafana/status.svg)](https://woodpecker.drycc.cc/drycc/grafana) -Drycc (pronounced DAY-iss) Workflow is an open source Platform as a Service (PaaS) that adds a developer-friendly layer to any [Kubernetes](http://kubernetes.io) cluster, making it easy to deploy and manage applications on your own servers. +Drycc (pronounced DAY-iss) Workflow is an open source Platform as a Service (PaaS) that adds a developer-friendly layer to any [Kubernetes][k8s-home] cluster, making it easy to deploy and manage applications on your own servers. For more information about the Drycc Workflow, please visit the main project page at https://github.com/drycc/workflow. We welcome your input! If you have feedback, please [submit an issue][issues]. If you'd like to participate in development, please read the "Development" section below and [submit a pull request][prs]. -# About -This repository aims to contain the necessary components for a production quality monitoring solution that runs on top of the kubernetes cluster scheduler. It provides part of the [TICK](https://influxdata.com/time-series-platform/) stack which is produced by the influxdata team. - -## Current State -Currently this repo provides only 2 components (Telegraf, and Grafana). Telegraf is the metrics collection agent that runs using the daemon set API. For more infomation please read [this](telegraf/README.md). - -Grafana is a stand alone graphing application. It natively supports Timescaledb as a datasource and provides a robust engine for creating dashboards on top of timeseries data. We provide a few out of the box dashboards for monitoring Drycc Workflow and Kubernetes but please feel free to use them as a starting point for creating your own dashboards. - -# Architecture Diagram - -``` - ┌─────────────┐ ┌──────────────┐ - │ Router │ │ Logger │ - └─────────────┘ └──────────────┘ - │ │ - logs file │ - │ │ - v v -┌──────────┐ ┌─────────────┐ logs/metrics ┌──────────────┐ -│ App Logs │───Log File──> │ Fluentbit │──────topics────>│ Redis Stream │ -└──────────┘ └─────────────┘ └──────────────┘ - │ -┌──────────┐ │ -│ HOST │───┐ │ -│ Telegraf │ │ │ -└──────────┘ │ │ - │ v -┌──────────┐ │ ┌─────────────┐ ┌─────────────┐ -│ HOST │───┼──────────>│ Timescaledb │<──────Wire──────│ Telegraf │ -│ Telegraf │ │ └─────────────┘ Protocol └─────────────┘ -└──────────┘ │ │ - │ │ -┌──────────┐ │ v -│ HOST │───┘ ┌─────────────┐ ┌─────────────┐ -│ Telegraf │ │ Grafana │<────────────────│ Prometheus │ -└──────────┘ └─────────────┘ └─────────────┘ - -``` +## Description + +[Grafana](https://grafana.org/) is a graphing application built for time series data. It natively supports prometheus and provides great dashboarding support. This project is focused on provided a grafana installation that can be run within a kubernetes installation. The grafana application is agnostic to [Workflow][Workflow] and can be installed as a stand alone system with the monitoring suite. + +## Configuration + +| ENV Var | Default Value | Description | +|---------|---------------|-------------| +| [APP_MODE](#app_mode) | development | Has two possible values (production or development). | +| [DATA_PATH](#data_path) | /var/lib/grafana | Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used). | +| [LOG_PATH](#log_path) | /var/log | Directory where grafana can store logs. | +| [SERVER_PROTOCOL](#server_protocol) | http | Protocol (http or https). | +| [BIND_ADDRESS](#bind_address) | `""` | The ip address to bind to, empty will bind to all interfaces | +| [BIND_PORT](#bind_port) | 3000 | The http port to use | +| [DOMAIN](#domain) | localhost | The public facing domain name used to access grafana from a browser | +| [ENFORCE_DOMAIN](#enforce_domain) | false | Redirect to correct domain if host header does not match domain. Prevents DNS rebinding attacks | +| [ROOT_URL](#root_url) | `"%(protocol)s://%(domain)s:%(http_port)s/"` | The full public facing url | +| [ROUTER_LOGGING](#router_logging) | false | Log web requests | +| [STATIC_ROOT_PATH](#static_root_path) | public | the path relative working path | +| [ENABLE_GZIP](#enable_gzip) | false | enable gzip | +| [CERT_FILE](#cert_file) | no default | https certs | +| [CERT_KEY](#cert_key) | no default | https certs key | +| [DATABASE_TYPE](#database_type) | no default | Potential values: mysql, postgres, sqlite3 | +| [DATABASE_HOST](#database_host) | no default | Address to external database host | +| [DATABASE_NAME](#database_name) | no default | Name of database to store information | +| [DATABASE_USER](#database_user) | no default | User to log into the database with | +| [DATABASE_PASSWORD](#database_password) | no default | Database password | +| [DATABASE_SSL_MODE](#database_ssl_mode) | no default | For "postgres" only, either "disable", "require" or "verify-full" | +| [DATABASE_PATH](#database_path) | no default | For "sqlite3" only, path relative to data_path setting | +| [SESSION_PROVIDER](#session_provider) | file | Either "memory", "file", "redis", "mysql", "postgres", default is "file" | +| [SESSION_PROVIDER_CONFIG](#session_provider_config) | no default | Provider config options
`memory: not have any config yet`
`file: session dir path, is relative to grafana data_path`
`redis: addr=127.0.0.1:6379,pool_size=100,db=grafana`
`mysql: user:password@tcp(127.0.0.1:3306)/database_name`
`postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable` | +| [SESSION_COOKIE_NAME](#session_cookie_name) | grafana_sess | Session cookie name | +| [SESSION_LIFE_TIME](#session_life_time) | 86400 | Session life time, default is 86400 | +| [GRAFANA_REPORTING](#grafana_reporting) | false | Server reporting, sends usage counters to stats.grafana.org every 24 hours. No ip addresses are being tracked, only simple counters to track running instances, dashboard and error counts. It is very helpful to us. Change this option to false to disable reporting. | +| [GOOGLE_ANALYTICS_UA_ID](#google_analytics_ua_id) | no default | Google Analytics universal tracking code, only enabled if you specify an id here | +| [DEFAULT_USER](#default_user) | admin | default admin user, created on startup | +| [DEFAULT_USER_PASSWORD](#default_user_password) | admin | default admin password, can be changed before first start of grafana, or in profile settings | +| [SECRET_KEY](#secret_key) | SW2YcwTIb9zpOOhoPsMm | used for signing | +| [LOGIN_REMEMBER_DAYS](#login_remember_day) | 7 | Auto-login remember days | +| [COOKIE_USERNAME](#cookie_username) | grafana_user | | +| [COOKIE_REMEMBER_NAME](#cookie_remember_name) | grafana_remember | | +| [DISABLE_GRAVATAR](#disable_gravatar) | false | disable gravatar profile images | +| [DATASOURCE_PROXY_WHITELIST](#datasource_proxy_whitelist) | no default | data source proxy whitelist (ip_or_domain:port seperated by spaces) | +| [ALLOW_SIGN_UP](#allow_sign_up) | true | allow user signup / registration | +| [ALLOW_ORG_CREATE](#allow_org_create) | true | Allow non admin users to create organizations | +| [AUTO_ASSIGN_ORG](#auto_assign_org) | true | Set to true to automatically assign new users to the default organization (id 1) | +| [AUTO_ASSIGN_ORG_ROLE](#auto_assign_org_role) | Viewer | Default role new users will be automatically assigned (if disabled above is set to true) | +| [LOGIN_HINT](#login_hint) | email or username | Background text for the user field on the login page | +| [ANON_ACCESS](#anon_access) | false | enable anonymous access | +| [ORG_NAME](#org_name) | Main Org. | specify organization name that should be used for unauthenticated users | +| [UNAUTHED_USER_ORG_ROLE](#unauthed_user_org_role) | Viewer | specify role for unauthenticated users | +| [GITHUB_AUTH](#github_auth) | no default | Enable Github Auth (true/false) | +| [GITHUB_AUTH_ALLOW_SIGN_UP](#github_auth_allow_sign_up) | false | Allow signup with github auth | +| [GITHUB_AUTH_CLIENT_ID](#github_auth_client_id) | no default | Github Client Auth | +| [GITHUB_AUTH_CLIENT_SECRET](#github_auth_client_secret) | no default | Github Auth client secret | +| [GITHUB_AUTH_SCOPES](#github_auth_scopes) | user:email,read:org | Github Auth Scopes | +| [GITHUB_AUTH_URL](#github_auth_url) | https://github.com/login/oauth/authorize | Github authorization URL | +| [GITHUB_AUTH_TOKEN_URL](#github_auth_token_url) | https://github.com/login/oauth/access_token | Github Auth Token URL | +| [GITHUB_AUTH_API_URL](#github_auth_api_url) | https://api.github.com/user | Github Auth API URL | +| [GITHUB_AUTH_TEAM_IDS](#github_auth_team_ids) | "" | Team IDs to use for Github Auth | +| [GITHUB_AUTH_ALLOWED_ORGS](#github_auth_allowed_orgs) | "" | Allowed ORGs for Github Auth | +| [GOOGLE_AUTH](#google_auth) | no default | Enable Google Auth for login (true/false) | +| [GOOGLE_AUTH_ALLOW_SIGN_UP](#google_auth_allow_sign_up) | false | Allow people to sign up using Google Auth | +| [GOOGLE_AUTH_CLIENT_ID](#google_auth_client_id) | no default | Google Auth Client ID | +| [GOOGLE_AUTH_CLIENT_SECRET](#google_auth_client_secret) | no default | Google Auth Client Secret | +| [GOOGLE_AUTH_SCOPES](#google_auth_scopes) | https://www.googleapis.com/auth/userinfo.profile
https://www.googleapis.com/auth/userinfo.email | Google Auth Scopes | +| [GOOGLE_AUTH_URL](#google_auth_url) | https://accounts.google.com/o/oauth2/auth | Google Auth URL | +| [GOOGLE_AUTH_TOKEN_URL](#google_auth_token_url) | https://accounts.google.com/o/oauth2/token | Google Auth Token URL | +| [GOOGLE_AUTH_API_URL](#google_auth_api_url) | https://www.googleapis.com/oauth2/v1/userinfo | Google Auth API URL | +| [GOOGLE_AUTH_ALLOWED_DOMAINS](#google_auth_allowed_domains) | "" | Google Auth allowed domains | +| [AUTH_PROXY](#auth_proxy) | no default | If enabled use an authorization proxy | +| [AUTH_HEADER_NAME](#auth_header_name) | X-WEBAUTH-USER | Header to use for authorization | +| [AUTH_HEADER_PROPERTY](#auth_header_property) | username | Property to use in Authorization Header | +| [AUTH_AUTO_SIGN_UP](#auth_auto_sign_up) | true | Auto Signup user with data passed from Authorization Header| +| [BASIC_AUTH](#basic_auth) | true | Enable Basic Auth | +| [LDAP_AUTH](#ldap_auth) | no default | Enable LDAP Auth | +| [LDAP_AUTH_CONFIG_FILE](#ldap_auth_config_file) | /etc/grafana/ldap.toml | Config file to use for LDAP Auth | +| [SMTP](#smtp) | no default | Enable SMTP/Emailing | +| [SMTP_HOST](#smtp_host) | no default | SMTP Host | +| [SMTP_USER](#smtp_user) | no default | User for SMTP | +| [SMTP_PASSWORD](#smtp_password) | no default | Password for SMTP | +| [SMTP_CERT_FILE](#smtp_cert_file) | no default | Cert file for SMTP HTTPS | +| [SMTP_KEY_FILE](#smtp_key_file) | no default | Key file for SMTP HTTPS | +| [SMTP_SKIP_VERIFY](#smtp_skip_verify) | false | Skip HTTPS verify | +| [SMTP_FROM_ADDRESS](#smtp_from_address) | admin@grafana.localhost | From address to use when sending emails | +| [WELCOME_EMAIL](#welcome_email) | false | Send welcome email on sign up | +| [LOG_MODE](#log_mode) | console | Either "console", "file", default is "console" Use comma to separate multiple modes, e.g. "console, file" | +| [BUFFER_LENGTH](#buffer_length) | 10000 | Buffer length of channel, keep it as it is if you don't know what it is. | +| [LOG_LEVEL](#log_level) | Info | Either "Trace", "Debug", "Info", "Warn", "Error", "Critical", default is "Trace" | +| [LOG_LEVEL_CONSOLE](#log_level_console) | Info | For "console" mode only | +| [LOG_LEVEL_FILE](#log_level_file) | Info | For "file" mode only | +| [LOG_ROTATE](#log_rotate) | true | This enables automated log rotate(switch of following options), default is true | +| [LOG_MAX_LINES](#log_max_lines) | 1000000 | Max line number of single file | +| [LOG_MAX_LINES_SHIFT](#log_max_lines_shift) | 28 | Max size shift of single file, default is 28 means 1 << 28, 256MB | +| [LOG_DAILY_ROTATE](#log_daily_rotate) | true | Segment log daily, default is true | +| [LOG_MAX_DAYS](#log_max_days) | 7 | Expired days of log file(delete after max days) | +| [AMQP_PUBLISHER](#amqp_publisher) | no default | Publish Events to AMQP | +| [AMQP_PUBLISHER_HOST](#amqp_publisher_host) | no default | RabbitMQ URL | +| [AMQP_PUBLISHER_EXCHANGE](#amqp_publisher_exchange) | no default | AMQP Publisher Exchange | +| [DASHBOARD_JSON](#dashboard_json) | true | Poll a location for json files that contain dashboards | +| [DASHBOARD_JSON_PATH](#dashboard_json_path) | /usr/share/grafana/dashboards | Location to scan for json dashboards | +| [PLUGINS_PATH](#plugins_path) | /var/lib/grafana/plugins | Path to where grafana can install plugins | +| [GF_INSTALL_PLUGINS](#gf_install_plugins) | no default | Pass the plugins as a comma seperated list | + +## Development + +The provided `Makefile` has various targets to help support building and publishing new images into a kubernetes cluster. + +### Environment variables + +There are a few key environment variables you should be aware of when interacting with the `make` targets. + +* `BUILD_TAG` - The tag provided to the podman image when it is built (defaults to the git-sha) +* `SHORT_NAME` - The name of the image (defaults to `grafana`) +* `DRYCC_REGISTRY` - This is the registry you are using (default `registry.drycc.cc`) +* `IMAGE_PREFIX` - This is the account for the registry you are using (default `drycc`) + +### Make targets + +* `make build` - Build container image +* `make push` - Push container image to a registry +* `make upgrade` - Replaces the running grafana instance with a new one + +The typical workflow will look something like this - `DRYCC_REGISTRY= IMAGE_PREFIX=foouser make build push upgrade` + [k8s-home]: http://kubernetes.io/ [issues]: https://github.com/drycc/monitor/issues [prs]: https://github.com/drycc/monitor/pulls -[v2.18]: https://github.com/drycc/workflow/releases/tag/v2.18.0 +[workflow]: https://github.com/drycc/workflow \ No newline at end of file diff --git a/charts/grafana/Chart.lock b/charts/grafana/Chart.lock new file mode 100644 index 0000000..3b97f14 --- /dev/null +++ b/charts/grafana/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: oci://registry.drycc.cc/charts + version: 1.1.3 +digest: sha256:4928b76b9f4927f43702c2617b139b2fe7298055b9b6d0c363a7a94530985595 +generated: "2024-11-18T16:21:05.150712088+08:00" diff --git a/charts/monitor/Chart.yaml b/charts/grafana/Chart.yaml similarity index 68% rename from charts/monitor/Chart.yaml rename to charts/grafana/Chart.yaml index 0707954..f3bffea 100644 --- a/charts/monitor/Chart.yaml +++ b/charts/grafana/Chart.yaml @@ -1,12 +1,12 @@ -name: monitor -home: https://github.com/drycc/monitor +name: grafana +home: https://github.com/drycc/grafana apiVersion: v2 appVersion: 1.0.0 dependencies: - name: common repository: oci://registry.drycc.cc/charts version: ~1.1.2 -description: Monitoring for Drycc Workflow. +description: Grafana for Drycc Workflow. maintainers: - name: Drycc Team email: engineering@drycc.com diff --git a/charts/grafana/charts/common-1.1.3.tgz b/charts/grafana/charts/common-1.1.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..d1035f1a56b79241a8ae1ff4bfca514e8ceb60c4 GIT binary patch literal 10632 zcmV;3DRDc zVQyr3R8em|NM&qo0PKBvf7>{gVEwI6fn{71D-}|b&mM1kc2Z7y+FO^`PNjBc(o+MG zkc2WtumDhwI`Mw?{ox|Po3bU@?lQZ7DVYQwfQNVE0r1IuPNKot3{vhb;XM3m{m*bX z93CAU$o~$9!{UF3dq=~sMhAz((QtThFg*HdINCcn9DN0b8|X{vNf?LptKq%dDh}?6 zJTS%|5M`J|Ct$JH3lQ@uj=B8%EnWIPIFl68n~^y>0W;2Hb}|@DF`p%4&nNRiAW97s z9rflAM;zh^QFhXE0h&V`o&b*E{9k1`&-Z%(L{PwbkKkOj(Nixp48Dxf2?+6+Lb~iN zOz%eC$lLEdG`{}P9?ky?7$%5qNC33)|M2*@$p6Qq!)N~A#`EF@$T9GX7vP)&%m78d zCzzrDjF;f-d~o&_a6&=`U=)Cu5+5;!nD@}#e}3-yX#VGD9*2-4Hux1zCOEwb@Y z6fxwU2Poorf)PCdZ!X@P{q>!@=M8(W2Di7po`3=tczyyH;$VV9B$W8+Tp-ts=zA&{nS%fKSF;4(+z;1wj4xDSW zbb#}hpMG+HqqLa=c04<;-!|C^ddwZ)o{rR)28f8tw>*zCNP~OLqX4kkc~mWkDn_k!pGE=U3Wdn$guX{~t`Y7&rMYaF zy?abtI%k5}9pET#*~kp4$x~y(6M%jPo@msxsYfoqc9R;O1rUV{<)JEiSp}-9k%Ox7 z-GF;eViJ<+@;kHw>a9xI1laE+mOG<2S5@^F*e}5sQ5hzo;LT)=XoNUo9wvhsVO%(F zPTMf0ppQvl$ZQ~fM;xzCb}<7ECl9HWpq!26&)o0tUZ6>!ie zPT>!Nd5eXLU#bwR>tAHUkwT#Wr1)8MG~(zJcfd~Psj)_DFuaqF7o41C6XO6c`;e8j zFku&$sOtkS_dTsuLPAB&Qki`1(&%jvLJ}AdrGj@?wRuQJkcOgrMtWz`b)I$oL^A7c zk?3Sa=FM#~*#(;Bauk+Fb2iVO{ssDd@C8IT5(X3)4fA4J3$%!Eu#4s~Us_Y#FBSzc z#SxzX2gb3NXU4SB3HVxeDkXY1Huw0sSAq|_FZRwjmGk9L%Tlgdult5ip& zhAqyLmG2|*>=OO5?UEZ|HQmWNkCBRY{t?KiEH9oO840zhep?!Gw3@z`mh$=(Va{&; zhgXD|4gW_`gsb~1Znda-ICb|Yn|(VAsr!ql3ITOTE!SB%6`I_zVCt&T9*iDp6xWj= zh(>+&hzX*D3R;`?*^cwCT3UneZ~0Q|bNwy7)aLm=bey;DDaZeP7{f6R*SG@E7XLpw zI4Z>dkA??F&+-3lJP#KC=Q9L2q*KJfcd0lBdbPh+G~|d=r~#e4kQs(}k{Y1Xkc?rN zgmAirvRnN036liU@;ma4o6NNt#xI~LCw43Dnk5*&}%*LZdlTFE>=X=@x5_{K5-Pu z)r@ZV(jCRo)VuzZw8qwnI-ip0H!|*sG*fz;v^5qsVmDNoqU%FX0MVR8tTXRrC!tNAh*Vrbi^w1vyNjREP#8^iJnbI}L zF*;foYxrbANK17nV*HzjA=v7M|M}hxS81KKD640zZCk}(e*5MOP{c?=ebkk!=otDd z$ea6{MXJGs&j`hThr;Mx358A7*N4ip%eP$-=+j^Y{N11QV2m@eUkP!)maQz zFx>Z7MW>EM_j4LnU~aF3E8TM(O$UoTS)>KIr3!UuZLPe%>G*&s$NpU>WAQv0t4sz1 zoF-Cli&;q+3cz1cI5)v8fEn2L_P_CZZ@>*)0#1NGL;iJ5=^M!8iU^K4B|+k&03;E| zVOn6{+xLdx2J=}V3M9a2EWZXfGwjcRKsCe>126(G2xM>{hQRrJd+PweI8A(>P~`RA z&oI;7!pv?PG5`TZQ}ju$hX5Qgj$i=DL@r`X>iYn0Z^0P(FkwhR#?e#~rx^D&S^~Q) zj=-Pyy?tQ{Bq0|)01!Ah;)qV5k0eGWB#8nCAWDg#?H{t5+R8oFtA3<;?19jvp+{IG=YH|2cI3`EUJK|M_3He|vR%yW@b} z9D2YPz{wS-0u$D{jU=BYYTI1=w;cbM=v`ax|5xq%fA;qdiu->Kj`yD9|66&s8UL4U zy`NE%OlOLV71Jj1l)^ypM8d?sYAHB|jNK3#Fr6C>_zZF&KuTdSMyj0G>;25&6rHHF ziv96G>Z$ft;> zKYoDm9Iep9QQE^&SF}7(lY~<6tlSfOSn0BF#f+3u z(9fp@36i=r`_D{+AchpG?0Xj!TLUi8V$}tvrU$+k9Uv2{v`~3Cnlhn;rS1^=`?V0{ zSnhgLq=z}UA<;_?_>7Vpqi{sKCa1v#XX&r3y$eWD#7&`kW2+6N*-~;5RWgIQu&py-*ILprXF9zPw8mBFK<;&VE+?$gMx^$UbVEo}2>bV|4N6>gt!DF5g~(cbAtx zU7mn{CyC<6n34q!5Ll9g0zaV=3mN|wm|W;+svCFe8$H!#S?;CtF#^TsN5YX34rqZQ z04E$#K#?$1U>M4=zXl`>fKQPWVlZT24wr%eB7_2E|7WNm2g~DfCjAlW? zWR8FzLOf?{>God|%4diI98CzFE9&%NB-$rqxraIckf%GY1L8ApjClm-Snj)?q6kqK z4rb)WRpjU1%y5X@bY66MJu~bML;Si> zNvG`Oij@iS3z$G0KGF1vC;yb~KRk!i_3i!qs@?uOI@~`f+J8q!&*wjE<=MLZXT>jK zN@5`Z1;NQPA*##|O^)(ESq`H7^O6v5B{pmOhtH5I4IGud;S>`>q=8YVoAd3ZR-2BW zErkUoQv}lj2ljz*EGy-5qPdr@;Bwg08X0d|CmhV$_Ak^CKy% zBsKj`RBl0mUmaqX(4S_Vx5U3j6r4+PPiN>#+WJ7W#=vfX zVv2moQE&oeC?-ee6h#4|7fBeZTFeA^`VGt&XT=LFnVS8zTseAO|81{d$Fp5nHrFf5 zgtp!b7Lwf)I4yF9O(%NWt<3;7j@&fOx75NZfN?D5E%(-|WSjh7xvXwIs9ZJuCEij; z@C5AIso55ozMPLDT22Z*%{BZ5BoY4Iq@}9^f8_4vLU(e#Ssxo9;VDBdWQGnW2%Fzn zJq@&8Vvw~{e8KZ=q4J9Uw9~r&YP?sWNX+lte47&W8I*cvE6c5cPfE)OJPvjILk&D1 z`EY<|L4!!#^Dz`G;sS98dg%z489!c>mBp-X9H*$H&9r zKzGoak{90}jNI=J4nR7j{F26h|JSZu(fFpn`_a?C{+IpJmEYgE{{h{|&kH%;W~&Z( zk{G-d8(B2^O=HlEa4Q4O>l%>8VW8F%sXqn30(JLYPtNu%W zyC6d1n{WEB;`+pewHjf;Ocx?NX?OxM0WbpS6K2x()FsW>?b@Q9wD#aU;vSH%gxUqW zgn}rUj}h$)kV22@Y92V};0A{w7$d+bjF{l#KxoGRPvr4a94z5H1OXBTis(hU+yV7^ z2q08+6Gt3PQE92;tw5Z;&hdVR($8i=-$;XRAX_)SGWT81Y*E?hZzt&qsTBYLhy^S=up?OQ zoB>BorBiA{6fMECKYq3{PX($BKpgQrw0`z&5)#Od4vODRbCOic4So?MR$)Ch$2~4o zdJd<;Le^O}LJkOP?fo)FK(1#Bde+$*DCi&3>j)pd{W4{TgH+e+F1x0lhWp1_&hH_TM@72>dl#ztD|@0*iDh^Bg)-5jNnwR zNa_V`7)6AuwS6f)-+zAv7@kHb=!pt=f_-6$0Wn0OPOJbwrQ519-7Idb~8czeLau8vRH{?|u8Y%%}qXn()7 z{&)Cn|8M2lzWr|*^NhnNfHY92lvGAxH8q-&*>Ojeha0I%<+Dhd&zmTXqA-SvQeyflm{SqzPM5MTy+N9WN^?>8%!h87LEDE?LdhW3N17B{zBj zC^zngDrV%MvOP0qW$)&OTUkCEaK45fYr8?Li9_a&lL+AXJmFAy%+hcZW}80)km;ST z9*dFhF?|M*eWs64p={yj3p8+H{y3l}Mzl^DD{^7)7X$*CU3UW`F8tJxkQk5!qCf%D z;y_2#b;{UWlvIAN?kB^Mj{xQ`<*^Y<5aMeTnm~f_i7A5D)(&9y+W5`vH9*|=dVeK1 zXo2W!1qjcDd8GR#x^zRa%q~MkVP&QJ)1E4+^3c#+%*h;a7(fm+>h#Rid&E^3UZ)s} zv46d3VN@%In&iaVWZ3GmXhT9}RRQs!wqe=+Q{!2q`PUl%JsysV_TS;?Isa=b&(`z5 z-hGONFcYFP(g9MrX-BTC)QU%&$=s}nJ_<+HmF#5nk0|-m0#S+sB)FU=kq?gc-u(bZ=ZL9RQcx+(f>7NMY4z2jxpb)^;B2B7VTTy#O^3b|i1CqEGN+XN!N z0E*%e`Up7RIKcVUx%;4CinvJdu%1xS61ptUMH`~$EZxq<$Q;>G z`7COr!WJQ(V;zL%Gt9uQM%CLNuS8iH36GKA_j>0OlMAkZq#M>O+~X|3E%7e5q>0Y! z-ZtZ06f6aj44s&dDXo|?wFNri-3}q`L^=nsGrd~X0aM=;F!izYharcJ^7}(vQDK5n03DD>%Cd266(5H>P;Au8+%ucBt;erZK1X~ z5rv)rsmJ3)hD)K@ZxEQlg&4#F zQ5Y66%7o;%5B=M8HcOsDO9o84TFB+Bq*hl2f^wiwi~2_!I^}vRs#aGYE>fiW%(jv# z#-+TIY-L^5k8BIe^x3PnXv(HvZBe_U83xSot4)QX)VW5>PD1C}I)VKzGDjVUevRbT zALm)HY(ubIqh7c|n`5O6b z|1w0;;tw#6P0F~nAwl%z6yPX8pPFjzq@AwcQ_#w621Glrr3W|=3Mk?fqXiO_CD%1V zieRt=zBJ{7PCS_%m!KBXtklljVHd__&hsE}^l1gbhy3*f#q%>p6=!PfIep-SD4Oz_ z14ctRheXVw%puBq@VVcO$)lR}OB4&-9Lc^^Gx;(-E)m?!5LJs)DULA%j1xibO}tQm zT)_V#!#? z&fCr>0dDDl5>aIWarjg8Bf6;@_FJtC#5ie!Ja9gdW|X z7}e9Kr45UbuL4eb2rt!#mpPH8I+Qgm#+G%j)#@l~yHp?Vq&+)11&#yK(*(3gD7?-X z-HNKp3`H;dWC~~Es(8Ix`oR=6z0i`@DnA+mPva^rtO>@D;djc6D(d=#P7?7pXxht;hJp(&A@+oe` zt2j97edZ{9biQUE9S{(tPQBC;Gc`-p_m7cd25tJ%bZC(2SM{u0yQQ-+*OaxkXMO?W zqXQKAHXc4(x3gI12^4lmqX>>e6yy)UFg$u{^IF}u-W`)XeqnKZ@dUVPx->?yL7zGN zO`_2;-EgcPRo&rcZ80x)MIPgHr!(xzqjJl0^P%DZG+{>O zJ#AAKCjW}lbR@eWxj_`{3bpeljAsyx`hE51kI5WS?1NDs>?UzsF7-v92KTE|$IY@l z^D!<->beSOiauScbb~ix_&w$_%I19h_=|-Ld2GuUVSX}B!C_b6_VFlAiixa5j^WrW znM^`d$$HfnFqH-_5cOj+FLkd0Wc8M$obwyo1Y5NYBw>hIMvL6ao!leZ>`l< zG_#)usnQ!Q9qZ+0t&u{~s@>XdjUCdr<-Y22tr1ex@m_wK%W(1qs`PnPkXcv{ixgyu z|MQ>PciY|Q$N>Z~MGH(4R`x`o?b5O?%&tndG5dpY?~2sjM%-q!1v*|Y4VR7SM{!E` z=AMf6e@-ZzqV1jkFxorbE3N+@?me&nZ{vBW^B?V_fUXq90lDY5Qs8Nl^hJSe<7s+4 zuaBx%C(K|%{~ydf3G5U+WGw10lS?1Z$=_rv7ZYqdulpz0-whr*!3Y7wRJAgj(3Sd$ngSQ z$@1>)ZR$ojS=7bpgI&F0QQ|l&RU=Ng(FT>}T~fjF=)H1=2>5$6rO}$^nVE8RzlIU@*_l{PT2{ z3?>5>Q#>6AOfhf9=iv*9J6Eus{MCswWXS^MHXdp!UybTN< zQXk2wah)q#4nMDS)MiQ|DK8pQ%Qm=zFM2brY_RMIs`sTWrH|WODeHf=0=CWlpQF8_ z(*Dnb<7fT9jc4onUmfO!BX#$jJW7WXFhxAe_0SR0df}x=R0n&fh-;2+j)R#?kMvtv z$2BiI?w6`**iypOWc^n6S+@6{yo(keAa!(xEBX`@%7Ks|2Egs@>CWzTGDbcRC2~xp z&f2!qxEH+;;j)%F^r37zwgp@Qeq$sOQvB5UEMvHyD51^(yA^u+3m8L&jt+n;PlYe4 z!fz8C1^`71uz*yhwy1nlsJ7v?PGkWoY8K7R%$IH&WpAIF0w^mUEWgN{=s3huwJ`BR zUJ)hSUp-)}hFotugADzpR8{Imw|I(lu?^cfuZ5N!99>sp7pRp#5A-YiWBF&c2v`fH zCRLMXmbhG`(pp{3VC&2^=1HX4lj^UPSgZs%i~5Su=&TWI20bUgK2W{%AypjzY5be` zuv1v4FPIrwOQjTE3yxcn@er$BcoZ!#QXBJx=%jkBiA=!Duh~l-=La|f#^l;*PHiYG zZ0wpPlB_d%3b8(%HZ57CD;oTOZ?4rd%A)!UjsoDg^yA0io1HIg2KNq+P8@X;$1iU# ze?0&3-%h|~HhK^sj(m;+uq$kE>#DHZ+Y|9dl)1g_zXsaIh|;?+)q&nB*(_WFio`Hc zq;uKfg5aQxZGF#nEHC?(S=uOCU`isjaZ+y2s=t9!;|5V?E}@j!^zxuISGVGdR$Qy3 z9aekR84v!HqL#WR!4S4uH|$Vw!c=m zFc}05vb3=}2Tv!@cLJo!79a%UpBbVT6ix7_PGGLl(o>ZRkW8+Rxq0M|k zxHKyWRffY(aqq9ZfugvJT;^Ex%}0$8btrv)w{R&Zo)x22tL!Hz4YzhPK?U(TZ5_)3 zU*&JrI|H&5nsyl=&p3)p_vIV76`K#L&o{N^uHgix)iE|g z3w!<>gD;6>k;y*_!}&Arq+yWM@F@nt08* z*-%!Sdn4VdC%yY*7&skf!Hx~q&xjdV1MYJ{Mj`oXHc3^5xbKU1vq?bB_ewaoRe`Y} zQd>b^3-ai>&mKg}oEN#QYC7{@{z3Af|EN#1|2Kye!(e>RP+*JycYIvB|KoUn|JncB z%JXpk-w%Se-hR7w@6XWeqw)RBKx;iextHvIUY}`WGd^DhN>}puOnnUwCVX~hr>_93 z+U0}PpJ9%Cp3urwv|@*>;zdXapVhmL1vo;{iV8Ju<0T=y#&x9DCnwix1stJ%O`3@X zBhjy0>$khF&Gj!)9AaORNZoMCCCX4AsY_g(rCPT*SFqkS)->XZDPFU8oa;e;G_sR6 zq1@%5ixRr7B4srNT^|| zdg`kv-+>6TS{p!9RiBzM_D2cibkTroi{-4*80gaPI|OqK*a~=e3FLGHv*s|48N>?n ze#a0_9h7?pa17W-9=@@~=_)*yGwL=NyxAbT<%Vz{KC?B_DvVWAZ_R-@!M`iaY-VOZ z$yv?1-!h}Nl&qRD`~6+Ra4qp45>3g5<3FR};X!f#=kfmY{*UcE4;cRuwDtDef1+sm zKi3}w(uDhXfmxy`+xLtG*-dRIAXG;0Y5^f#e=Px_Jgl04P;(e3527ZF(;8GN z45~h;^wRT(MHwmA;%V@=dlhQ>^7)ri=S5IcE z#7pd6wD_VMh<>RUgx!Tl85Yx`?hg9rDTjKJxRJjdR;8XN12y-DM1ycV~yL%;756VPCv zcI-_LVug9XYwWEC%6(&R8tfyFy>*Vz)LGD*jL@vv_lTbRaTs5^vdHX0Iba!H=< z)Z9NVYNl-SanXkk@U%QF@jrRW@qIUfw&i~v9hCCFM#Ja*pIdnzF#fl^`aj>VKk%13 zm;2$vesYi-&HR!rY$(iEMCNKiK3RJ$AwC<@9W%aiow;+qmks4P+v_>otL))Rj{XA# zwH`XxDz3H~e5XNTM>)w$I#aB-}pR3x8esZ(fJw%vF!%gLJ-SugX|HOpx zDMj}T0=49SjgE`)pS|JedH>&bo`;M7TqxSwL=ea#+eZ@#%KB6v4$8{hFD7KQwVA+B zHG)@)4p|`Xc??Viq{RqPgZTG7ks*v{PEz1HOEp1r7QbTr8b!(M@6)&%B9x$3x zE>?-zdN;pZyxy;^;EgpMF;nhEKD?qaWKHBsRA|p)HhcLvLPZnB`Dj4)ssV*Y;_Ng7 zn+-2CkL)o88t!I32gt_(N-u>(qGLiM>dV4(Kbv%3nb;DBJafZY?hx*4v~`sM>4vS0K=0Bkc*Z)t<6BxqP=cr&eF8V@Un{a2ovS z+BQ}Nyyj4qHN+Ju=59f&20-_WT-lTLxWiW_*ikYGSDb5&!v zIb9HKmmtzsF0*C>-2$7(;*!ajX@xMCRozrFswM4`7OHM>1CqoxR_`WC^5*UnrsB%+ ztWUbxh+a|B6`{6(TC=>fTZ`{5@+y7XCrmE2V`?Fp8j^?$Y;My-Q@K^+$IbCyiUNH9 ziy&I^|BeoZ#rxlmhx>=m@!xGc>xuu~DFA#ah-)+3e-+(6n%J-GPJP%{e!gGCSGBO2 zU~dVT-NLe}TDOR7ZNgm+$TeCrFP3EsfaH}EkmA3i-~*zpCPbG5W>xVPs*a;BKdO&m zn;Ud@v*b5<2U1}iPE`^`zqMAm0{XpjXeKmR7jCb$tGncZ6@$gqSzzTP!2mNm__z|P z(hy=hlI)_L(Ua@t&kVgu_-yt0)Fj;A$AqDamAd&TqWRWUg&BtKlS!Dvc6|gj9lPki zXT0F1TJWhF#yveV z1@uvdoRr2>mq!{$OpU%Xre#D%E6xr2SZlLF>eYVzG^;xHoHe8VY=-=6maf7`e9Qhz zh%cGJC{y;F5#Wj^!LGI$z`vY=(LulSOjMd3_n`kucvi5>);vb2=hfHl3N}|?=|ZQ* z_7V-~dqD4~JBqz+Hb3yBu6dhd1$nnH<3oF45BD_t|F#yodlaC>|35h1KPu*b?HxYv z|J%y*aQR=_9TsJzqd!JvUtihovb<#MGZIaPLm^ytizZ)s%ddRp)cN{pGJ`ge+~H80 z$u+Hw*4x@BZLN#RS7+&|dyXq{Q#FM0oyUb~0I{X)%_ysb9PelP9&1jFB_ewrO*JSS zSr&@7g!3@B+c9(YYT_xGSy3BLscx5nDT)w<90h~!+bs39?``d?VtQ=4GXdnVx{3O{ zhYogqrmEBx(Ow!I%Q;N#)WdKzEx8LB4SVTC*ZM{JQv8bhq`R57DV3Q!<{P=aeN44u zaf(+Iw6f%U>Dpy8YuNhv%rx6&nu4oQf)3pL@Jvy80j@PZE08Qk7)uXi;;q7zdEP0; zEbn?Vd&Y5fn=6$`Z3bq=s2snK;Ae9_iVie*hrlmTp$x83ZyJ+zNQaeeqrW=_L1KwxuvagCL(mweZ zeT+;t;@9kBRN?8U##PV7%xMgG{hCCtd;~y&fB7|gSr%*Q1{1;FhP}y0))bdz&OESAmnch{cAWy%ll9%%%hJkNJ}=qCo~|4E^gQ zmR<|%z4F*i+J0RR7GkE;~`GywoIRHi=w literal 0 HcmV?d00001 diff --git a/charts/monitor/templates/_helpers.tmpl b/charts/grafana/templates/_helpers.tmpl similarity index 62% rename from charts/monitor/templates/_helpers.tmpl rename to charts/grafana/templates/_helpers.tmpl index f03bd19..3fb8956 100644 --- a/charts/monitor/templates/_helpers.tmpl +++ b/charts/grafana/templates/_helpers.tmpl @@ -1,4 +1,4 @@ -{{/* Generate monitor grafana envs */}} +{{/* Generate grafana envs */}} {{- define "grafana.envs" }} env: - name: NAMESPACE @@ -27,25 +27,25 @@ env: {{- end }} - name: "BIND_PORT" value: "3000" -{{- if not (.Values.grafana.environment.DEFAULT_USER) }} +{{- if not (.Values.environment.DEFAULT_USER) }} - name: "DEFAULT_USER" value: {{ randAlphaNum 32 }} {{- end}} -{{- if not (.Values.grafana.environment.DEFAULT_USER_PASSWORD) }} +{{- if not (.Values.environment.DEFAULT_USER_PASSWORD) }} - name: "DEFAULT_USER_PASSWORD" value: {{ randAlphaNum 32 }} {{- end}} - name: "KUBERNETES_CLUSTER_DOMAIN" value: {{.Values.global.clusterDomain}} -{{- range $key, $value := .Values.grafana.environment }} +{{- range $key, $value := .Values.environment }} - name: {{ $key }} value: {{ $value | quote }} {{- end }} - name: "DRYCC_GRAFANA_URL" {{- if .Values.global.certManagerEnabled }} - value: https://drycc-monitor-grafana.{{ .Values.global.platformDomain }} + value: https://drycc-grafana.{{ .Values.global.platformDomain }} {{- else }} - value: http://drycc-monitor-grafana.{{ .Values.global.platformDomain }} + value: http://drycc-grafana.{{ .Values.global.platformDomain }} {{- end }} {{- if eq .Values.global.passportLocation "on-cluster"}} - name: "DRYCC_PASSPORT_URL" @@ -66,11 +66,11 @@ env: key: drycc-passport-grafana-secret {{- else }} - name: DRYCC_PASSPORT_URL - value: "{{ .Values.grafana.passportUrl }}" + value: "{{ .Values.passportUrl }}" - name: DRYCC_PASSPORT_KEY - value: "{{ .Values.grafana.passportKey }}" + value: "{{ .Values.passportKey }}" - name: DRYCC_PASSPORT_SECRET - value: "{{ .Values.grafana.passportSecret }}" + value: "{{ .Values.passportSecret }}" {{- end }} - name: "PROMETHEUS_LOCATION" value: {{ .Values.global.prometheusLocation }} @@ -87,35 +87,3 @@ env: - name: "REGISTRY_LOCATION" value: {{ .Values.global.registryLocation }} {{- end }} - -{{/* Generate monitor telegraf envs */}} -{{- define "telegraf.envs" }} -env: -- name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace -- name: "AGENT_QUIET" - value: "true" -- name: "AGENT_BUFFER_LIMIT" - value: "100000" -- name: "ENABLE_KUBERNETES" - value: "true" -{{- if (.Values.telegraf.postgresqlConnection) }} -- name: POSTGRESQL_CONNECTION - value: {{ .Values.telegraf.postgresqlConnection }} -{{- else if eq .Values.global.timeseriesLocation "on-cluster" }} -- name: PG_USER - valueFrom: - secretKeyRef: - name: timeseries-creds - key: user -- name: PG_PASSWORD - valueFrom: - secretKeyRef: - name: timeseries-creds - key: password -- name: POSTGRESQL_CONNECTION - value: "postgres://$(PG_USER):$(PG_PASSWORD)@drycc-timeseries.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:5432/monitor" -{{- end }} -{{- end }} diff --git a/charts/monitor/templates/monitor-grafana-deployment.yaml b/charts/grafana/templates/grafana-deployment.yaml similarity index 60% rename from charts/monitor/templates/monitor-grafana-deployment.yaml rename to charts/grafana/templates/grafana-deployment.yaml index fef0f9a..eb99314 100644 --- a/charts/monitor/templates/monitor-grafana-deployment.yaml +++ b/charts/grafana/templates/grafana-deployment.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: drycc-monitor-grafana + name: drycc-grafana labels: heritage: drycc annotations: @@ -16,49 +16,49 @@ spec: type: RollingUpdate selector: matchLabels: - app: drycc-monitor-grafana + app: drycc-grafana template: metadata: labels: {{- include "common.labels.standard" . | nindent 8 }} - app: drycc-monitor-grafana + app: drycc-grafana spec: affinity: - podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.grafana.podAffinityPreset.type "component" "" "extraMatchLabels" .Values.grafana.podAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }} - podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.grafana.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.grafana.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }} - nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.grafana.nodeAffinityPreset.type "key" .Values.grafana.nodeAffinityPreset.key "values" .Values.grafana.nodeAffinityPreset.values ) | nindent 10 }} + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset.type "component" "" "extraMatchLabels" .Values.podAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values ) | nindent 10 }} securityContext: fsGroup: 1001 runAsGroup: 1001 runAsUser: 1001 containers: - - name: drycc-monitor-grafana + - name: drycc-grafana image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/grafana:{{.Values.imageTag}} imagePullPolicy: {{.Values.imagePullPolicy}} {{- if .Values.diagnosticMode.enabled }} command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }} args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }} {{- end }} - {{- if or (.Values.grafana.limitsCpu) (.Values.grafana.limitsMemory)}} + {{- if or (.Values.limitsCpu) (.Values.limitsMemory)}} resources: limits: - {{- if (.Values.grafana.limitsCpu) }} - cpu: {{.Values.grafana.limitsCpu}} + {{- if (.Values.limitsCpu) }} + cpu: {{.Values.limitsCpu}} {{- end}} - {{- if (.Values.grafana.limitsMemory) }} - memory: {{.Values.grafana.limitsMemory}} + {{- if (.Values.limitsMemory) }} + memory: {{.Values.limitsMemory}} {{- end}} {{- end}} ports: - containerPort: 3000 - name: ui + name: grafana {{- include "grafana.envs" . | indent 8 }} - {{- if .Values.grafana.persistence.enabled }} + {{- if .Values.persistence.enabled }} volumeMounts: - name: grafana-data mountPath: /var/lib/grafana volumes: - name: grafana-data persistentVolumeClaim: - claimName: drycc-monitor-grafana + claimName: drycc-grafana {{- end }} {{- end }} diff --git a/charts/grafana/templates/grafana-pvc.yaml b/charts/grafana/templates/grafana-pvc.yaml new file mode 100644 index 0000000..2032faa --- /dev/null +++ b/charts/grafana/templates/grafana-pvc.yaml @@ -0,0 +1,24 @@ +{{- if .Values.persistence.enabled }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: drycc-grafana + labels: + heritage: drycc +spec: + accessModes: + - {{ .Values.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} +{{- if .Values.persistence.storageClass }} +{{- if (eq "-" .Values.persistence.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.persistence.storageClass }}" +{{- end }} +{{- end }} +{{- if .Values.persistence.volumeName }} + volumeName: "{{ .Values.persistence.volumeName }}" +{{- end }} +{{- end }} diff --git a/charts/monitor/templates/monitor-grafana-svc.yaml b/charts/grafana/templates/grafana-svc.yaml similarity index 57% rename from charts/monitor/templates/monitor-grafana-svc.yaml rename to charts/grafana/templates/grafana-svc.yaml index 23cf24d..a91eda1 100644 --- a/charts/monitor/templates/monitor-grafana-svc.yaml +++ b/charts/grafana/templates/grafana-svc.yaml @@ -2,19 +2,19 @@ apiVersion: v1 kind: Service metadata: - name: drycc-monitor-grafana + name: drycc-grafana annotations: - {{- with .Values.grafana.service.annotations }} + {{- with .Values.service.annotations }} {{- toYaml . | nindent 4 }} {{- end }} labels: heritage: drycc - app: drycc-monitor-grafana + app: drycc-grafana spec: ports: - port: 80 - name: ui - targetPort: ui + name: grafana + targetPort: grafana selector: - app: drycc-monitor-grafana + app: drycc-grafana {{- end }} diff --git a/charts/monitor/values.yaml b/charts/grafana/values.yaml similarity index 58% rename from charts/monitor/values.yaml rename to charts/grafana/values.yaml index 38025ae..b22ecf8 100644 --- a/charts/monitor/values.yaml +++ b/charts/grafana/values.yaml @@ -18,63 +18,43 @@ diagnosticMode: args: - infinity -grafana: - # limitsCpu: "100m" - # limitsMemory: "50Mi" - nodeAffinityPreset: - key: "drycc.cc/node" - type: "soft" - values: - - "true" +# limitsCpu: "100m" +# limitsMemory: "50Mi" - podAffinityPreset: - type: "" - extraMatchLabels: - security: "drycc-security" +nodeAffinityPreset: + key: "drycc.cc/node" + type: "soft" + values: + - "true" - podAntiAffinityPreset: - type: "soft" - extraMatchLabels: - app: "drycc-monitor-grafana" +podAffinityPreset: + type: "" + extraMatchLabels: + security: "drycc-security" - # Service - service: - # Provide any additional service annotations - annotations: {} +podAntiAffinityPreset: + type: "soft" + extraMatchLabels: + app: "drycc-grafana" - persistence: - enabled: false - accessMode: ReadWriteOnce - size: 5Gi - # Any custom grafana environment variables - # can be specified as key-value pairs under environment - # this is usually a non required setting. - environment: - ALLOW_SIGN_UP: false - # The passportUrl, passportKey and passportSecret are will no longer use the built-in passport component - passportUrl: "" - passportKey: "" - passportSecret: "" +# Service +service: + # Provide any additional service annotations + annotations: {} -telegraf: - # limitsCpu: "100m" - # limitsMemory: "50Mi" - postgresqlConnection: "" - nodeAffinityPreset: - key: "drycc.cc/node" - type: "soft" - values: - - "true" - - podAffinityPreset: - type: "" - extraMatchLabels: - security: "drycc-security" - - podAntiAffinityPreset: - type: "soft" - extraMatchLabels: - app: "drycc-monitor-telegraf" +persistence: + enabled: false + accessMode: ReadWriteOnce + size: 5Gi +# Any custom grafana environment variables +# can be specified as key-value pairs under environment +# this is usually a non required setting. +environment: + ALLOW_SIGN_UP: false +# The passportUrl, passportKey and passportSecret are will no longer use the built-in passport component +passportUrl: "" +passportKey: "" +passportSecret: "" redis: replicas: 1 @@ -86,14 +66,14 @@ global: # Valid values are: # - on-cluster: Run component within the Kubernetes cluster # - off-cluster: component is running outside of the cluster and credentials and connection information will be provided. - prometheusLocation: "on-cluster" - grafanaLocation: "on-cluster" redisLocation: "on-cluster" - rabbitmqLocation: "on-cluster" + grafanaLocation: "on-cluster" storageLocation: "on-cluster" + rabbitmqLocation: "on-cluster" databaseLocation: "on-cluster" passportLocation: "on-cluster" registryLocation: "on-cluster" + prometheusLocation: "on-cluster" # Role-Based Access Control for Kubernetes >= 1.5 rbac: false ingressClass: "" diff --git a/charts/monitor/templates/monitor-clusterrole.yaml b/charts/monitor/templates/monitor-clusterrole.yaml deleted file mode 100644 index 4886b1b..0000000 --- a/charts/monitor/templates/monitor-clusterrole.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if (.Values.global.rbac) -}} -# Define global role with the default system:aggregate-to-view cluster role and the two rules we just created -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ printf "%s:drycc-monitor" .Release.Namespace | quote }} -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.authorization.k8s.io/aggregate-view-telegraf-stats: "true" - - matchLabels: - rbac.authorization.k8s.io/aggregate-to-view: "true" -rules: [] # Rules are automatically filled in by the controller manager. -{{- end -}} diff --git a/charts/monitor/templates/monitor-grafana-pvc.yaml b/charts/monitor/templates/monitor-grafana-pvc.yaml deleted file mode 100644 index ea1059a..0000000 --- a/charts/monitor/templates/monitor-grafana-pvc.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{{- if .Values.grafana.persistence.enabled }} -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: drycc-monitor-grafana - labels: - heritage: drycc -spec: - accessModes: - - {{ .Values.grafana.persistence.accessMode | quote }} - resources: - requests: - storage: {{ .Values.grafana.persistence.size | quote }} -{{- if .Values.grafana.persistence.storageClass }} -{{- if (eq "-" .Values.grafana.persistence.storageClass) }} - storageClassName: "" -{{- else }} - storageClassName: "{{ .Values.grafana.persistence.storageClass }}" -{{- end }} -{{- end }} -{{- if .Values.grafana.persistence.volumeName }} - volumeName: "{{ .Values.grafana.persistence.volumeName }}" -{{- end }} -{{- end }} diff --git a/charts/monitor/templates/monitor-telegraf-clusterrole.yaml b/charts/monitor/templates/monitor-telegraf-clusterrole.yaml deleted file mode 100644 index 21eb73c..0000000 --- a/charts/monitor/templates/monitor-telegraf-clusterrole.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if (.Values.global.rbac) -}} -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ printf "%s:drycc-monitor-telegraf" .Release.Namespace | quote }} - labels: - rbac.authorization.k8s.io/aggregate-view-telegraf-stats: "true" - app: drycc-monitor-telegraf - heritage: drycc -rules: - - apiGroups: ["metrics.k8s.io"] - resources: ["pods"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["pods", "persistentvolumes", "nodes", "nodes/proxy", "nodes/stats"] - verbs: ["get", "list", "watch"] -{{- end -}} diff --git a/charts/monitor/templates/monitor-telegraf-clusterrolebinding.yaml b/charts/monitor/templates/monitor-telegraf-clusterrolebinding.yaml deleted file mode 100644 index 18e278f..0000000 --- a/charts/monitor/templates/monitor-telegraf-clusterrolebinding.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{- if (.Values.global.rbac) -}} -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ printf "%s:drycc-monitor-telegraf" .Release.Namespace | quote }} - labels: - app: drycc-monitor-telegraf - heritage: drycc -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ printf "%s:drycc-monitor-telegraf" .Release.Namespace | quote }} -subjects: -- kind: ServiceAccount - name: drycc-monitor-telegraf - namespace: {{ .Release.Namespace }} -{{- end -}} - diff --git a/charts/monitor/templates/monitor-telegraf-daemon.yaml b/charts/monitor/templates/monitor-telegraf-daemon.yaml deleted file mode 100644 index 6f679a0..0000000 --- a/charts/monitor/templates/monitor-telegraf-daemon.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: drycc-monitor-telegraf - labels: - heritage: drycc - annotations: - component.drycc.cc/version: {{ .Values.imageTag }} -spec: - selector: - matchLabels: - app: drycc-monitor-telegraf - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: {{- include "common.labels.standard" . | nindent 8 }} - app: drycc-monitor-telegraf - spec: - securityContext: - fsGroup: 1001 - runAsGroup: 1001 - runAsUser: 1001 - serviceAccount: drycc-monitor-telegraf - initContainers: - - name: drycc-monitor-telegraf-init - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest - imagePullPolicy: {{.Values.imagePullPolicy}} - args: - - netcat - - -v - - -u - - $(POSTGRESQL_CONNECTION) - {{- include "telegraf.envs" . | indent 8 }} - containers: - - name: drycc-monitor-telegraf - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/telegraf:{{.Values.imageTag}} - imagePullPolicy: {{.Values.imagePullPolicy}} - {{- if .Values.diagnosticMode.enabled }} - command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }} - args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }} - {{- end }} - {{- if or (.Values.telegraf.limitsCpu) (.Values.telegraf.limitsMemory)}} - resources: - limits: - {{- if (.Values.telegraf.limitsCpu) }} - cpu: {{.Values.telegraf.limitsCpu}} - {{- end}} - {{- if (.Values.telegraf.limitsMemory) }} - memory: {{.Values.telegraf.limitsMemory}} - {{- end}} - {{- end}} - {{- include "telegraf.envs" . | indent 8 }} diff --git a/charts/monitor/templates/monitor-telegraf-service-account.yaml b/charts/monitor/templates/monitor-telegraf-service-account.yaml deleted file mode 100644 index ffa2622..0000000 --- a/charts/monitor/templates/monitor-telegraf-service-account.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: drycc-monitor-telegraf - labels: - heritage: drycc diff --git a/deploy.mk b/deploy.mk deleted file mode 100644 index 2864d5d..0000000 --- a/deploy.mk +++ /dev/null @@ -1,9 +0,0 @@ -install: - helm upgrade monitor ../charts/monitor --install --namespace drycc --set grafana.org=${IMAGE_PREFIX},grafana.image_tag=${VERSION},telegraf.org=${IMAGE_PREFIX},telegraf.image_tag=${VERSION} - - -upgrade: - helm upgrade monitor ../charts/monitor --namespace drycc --set grafana.org=${IMAGE_PREFIX},grafana.image_tag=${VERSION},telegraf.org=${IMAGE_PREFIX},telegraf.image_tag=${VERSION} - -uninstall: - helm delete monitor \ No newline at end of file diff --git a/grafana/Makefile b/grafana/Makefile deleted file mode 100644 index 8195e90..0000000 --- a/grafana/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -SHORT_NAME ?= grafana -SHELL_SCRIPTS = rootfs/usr/share/grafana/start-grafana - -include ../default.mk diff --git a/grafana/README.md b/grafana/README.md deleted file mode 100644 index 786b68d..0000000 --- a/grafana/README.md +++ /dev/null @@ -1,122 +0,0 @@ -# Grafana - -## Description -[Grafana](https://grafana.org/) is a graphing application built for time series data. It natively supports prometheus and provides great dashboarding support. This project is focused on provided a grafana installation that can be run within a kubernetes installation. The grafana application is agnostic to [Workflow](https://github.com/drycc/workflow) and can be installed as a stand alone system with the monitoring suite. - -## Configuration -| ENV Var | Default Value | Description | -|---------|---------------|-------------| -| [APP_MODE](#app_mode) | development | Has two possible values (production or development). | -| [DATA_PATH](#data_path) | /var/lib/grafana | Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used). | -| [LOG_PATH](#log_path) | /var/log | Directory where grafana can store logs. | -| [SERVER_PROTOCOL](#server_protocol) | http | Protocol (http or https). | -| [BIND_ADDRESS](#bind_address) | `""` | The ip address to bind to, empty will bind to all interfaces | -| [BIND_PORT](#bind_port) | 3000 | The http port to use | -| [DOMAIN](#domain) | localhost | The public facing domain name used to access grafana from a browser | -| [ENFORCE_DOMAIN](#enforce_domain) | false | Redirect to correct domain if host header does not match domain. Prevents DNS rebinding attacks | -| [ROOT_URL](#root_url) | `"%(protocol)s://%(domain)s:%(http_port)s/"` | The full public facing url | -| [ROUTER_LOGGING](#router_logging) | false | Log web requests | -| [STATIC_ROOT_PATH](#static_root_path) | public | the path relative working path | -| [ENABLE_GZIP](#enable_gzip) | false | enable gzip | -| [CERT_FILE](#cert_file) | no default | https certs | -| [CERT_KEY](#cert_key) | no default | https certs key | -| [DATABASE_TYPE](#database_type) | no default | Potential values: mysql, postgres, sqlite3 | -| [DATABASE_HOST](#database_host) | no default | Address to external database host | -| [DATABASE_NAME](#database_name) | no default | Name of database to store information | -| [DATABASE_USER](#database_user) | no default | User to log into the database with | -| [DATABASE_PASSWORD](#database_password) | no default | Database password | -| [DATABASE_SSL_MODE](#database_ssl_mode) | no default | For "postgres" only, either "disable", "require" or "verify-full" | -| [DATABASE_PATH](#database_path) | no default | For "sqlite3" only, path relative to data_path setting | -| [SESSION_PROVIDER](#session_provider) | file | Either "memory", "file", "redis", "mysql", "postgres", default is "file" | -| [SESSION_PROVIDER_CONFIG](#session_provider_config) | no default | Provider config options
`memory: not have any config yet`
`file: session dir path, is relative to grafana data_path`
`redis: addr=127.0.0.1:6379,pool_size=100,db=grafana`
`mysql: user:password@tcp(127.0.0.1:3306)/database_name`
`postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable` | -| [SESSION_COOKIE_NAME](#session_cookie_name) | grafana_sess | Session cookie name | -| [SESSION_LIFE_TIME](#session_life_time) | 86400 | Session life time, default is 86400 | -| [GRAFANA_REPORTING](#grafana_reporting) | false | Server reporting, sends usage counters to stats.grafana.org every 24 hours. No ip addresses are being tracked, only simple counters to track running instances, dashboard and error counts. It is very helpful to us. Change this option to false to disable reporting. | -| [GOOGLE_ANALYTICS_UA_ID](#google_analytics_ua_id) | no default | Google Analytics universal tracking code, only enabled if you specify an id here | -| [DEFAULT_USER](#default_user) | admin | default admin user, created on startup | -| [DEFAULT_USER_PASSWORD](#default_user_password) | admin | default admin password, can be changed before first start of grafana, or in profile settings | -| [SECRET_KEY](#secret_key) | SW2YcwTIb9zpOOhoPsMm | used for signing | -| [LOGIN_REMEMBER_DAYS](#login_remember_day) | 7 | Auto-login remember days | -| [COOKIE_USERNAME](#cookie_username) | grafana_user | | -| [COOKIE_REMEMBER_NAME](#cookie_remember_name) | grafana_remember | | -| [DISABLE_GRAVATAR](#disable_gravatar) | false | disable gravatar profile images | -| [DATASOURCE_PROXY_WHITELIST](#datasource_proxy_whitelist) | no default | data source proxy whitelist (ip_or_domain:port seperated by spaces) | -| [ALLOW_SIGN_UP](#allow_sign_up) | true | allow user signup / registration | -| [ALLOW_ORG_CREATE](#allow_org_create) | true | Allow non admin users to create organizations | -| [AUTO_ASSIGN_ORG](#auto_assign_org) | true | Set to true to automatically assign new users to the default organization (id 1) | -| [AUTO_ASSIGN_ORG_ROLE](#auto_assign_org_role) | Viewer | Default role new users will be automatically assigned (if disabled above is set to true) | -| [LOGIN_HINT](#login_hint) | email or username | Background text for the user field on the login page | -| [ANON_ACCESS](#anon_access) | false | enable anonymous access | -| [ORG_NAME](#org_name) | Main Org. | specify organization name that should be used for unauthenticated users | -| [UNAUTHED_USER_ORG_ROLE](#unauthed_user_org_role) | Viewer | specify role for unauthenticated users | -| [GITHUB_AUTH](#github_auth) | no default | Enable Github Auth (true/false) | -| [GITHUB_AUTH_ALLOW_SIGN_UP](#github_auth_allow_sign_up) | false | Allow signup with github auth | -| [GITHUB_AUTH_CLIENT_ID](#github_auth_client_id) | no default | Github Client Auth | -| [GITHUB_AUTH_CLIENT_SECRET](#github_auth_client_secret) | no default | Github Auth client secret | -| [GITHUB_AUTH_SCOPES](#github_auth_scopes) | user:email,read:org | Github Auth Scopes | -| [GITHUB_AUTH_URL](#github_auth_url) | https://github.com/login/oauth/authorize | Github authorization URL | -| [GITHUB_AUTH_TOKEN_URL](#github_auth_token_url) | https://github.com/login/oauth/access_token | Github Auth Token URL | -| [GITHUB_AUTH_API_URL](#github_auth_api_url) | https://api.github.com/user | Github Auth API URL | -| [GITHUB_AUTH_TEAM_IDS](#github_auth_team_ids) | "" | Team IDs to use for Github Auth | -| [GITHUB_AUTH_ALLOWED_ORGS](#github_auth_allowed_orgs) | "" | Allowed ORGs for Github Auth | -| [GOOGLE_AUTH](#google_auth) | no default | Enable Google Auth for login (true/false) | -| [GOOGLE_AUTH_ALLOW_SIGN_UP](#google_auth_allow_sign_up) | false | Allow people to sign up using Google Auth | -| [GOOGLE_AUTH_CLIENT_ID](#google_auth_client_id) | no default | Google Auth Client ID | -| [GOOGLE_AUTH_CLIENT_SECRET](#google_auth_client_secret) | no default | Google Auth Client Secret | -| [GOOGLE_AUTH_SCOPES](#google_auth_scopes) | https://www.googleapis.com/auth/userinfo.profile
https://www.googleapis.com/auth/userinfo.email | Google Auth Scopes | -| [GOOGLE_AUTH_URL](#google_auth_url) | https://accounts.google.com/o/oauth2/auth | Google Auth URL | -| [GOOGLE_AUTH_TOKEN_URL](#google_auth_token_url) | https://accounts.google.com/o/oauth2/token | Google Auth Token URL | -| [GOOGLE_AUTH_API_URL](#google_auth_api_url) | https://www.googleapis.com/oauth2/v1/userinfo | Google Auth API URL | -| [GOOGLE_AUTH_ALLOWED_DOMAINS](#google_auth_allowed_domains) | "" | Google Auth allowed domains | -| [AUTH_PROXY](#auth_proxy) | no default | If enabled use an authorization proxy | -| [AUTH_HEADER_NAME](#auth_header_name) | X-WEBAUTH-USER | Header to use for authorization | -| [AUTH_HEADER_PROPERTY](#auth_header_property) | username | Property to use in Authorization Header | -| [AUTH_AUTO_SIGN_UP](#auth_auto_sign_up) | true | Auto Signup user with data passed from Authorization Header| -| [BASIC_AUTH](#basic_auth) | true | Enable Basic Auth | -| [LDAP_AUTH](#ldap_auth) | no default | Enable LDAP Auth | -| [LDAP_AUTH_CONFIG_FILE](#ldap_auth_config_file) | /etc/grafana/ldap.toml | Config file to use for LDAP Auth | -| [SMTP](#smtp) | no default | Enable SMTP/Emailing | -| [SMTP_HOST](#smtp_host) | no default | SMTP Host | -| [SMTP_USER](#smtp_user) | no default | User for SMTP | -| [SMTP_PASSWORD](#smtp_password) | no default | Password for SMTP | -| [SMTP_CERT_FILE](#smtp_cert_file) | no default | Cert file for SMTP HTTPS | -| [SMTP_KEY_FILE](#smtp_key_file) | no default | Key file for SMTP HTTPS | -| [SMTP_SKIP_VERIFY](#smtp_skip_verify) | false | Skip HTTPS verify | -| [SMTP_FROM_ADDRESS](#smtp_from_address) | admin@grafana.localhost | From address to use when sending emails | -| [WELCOME_EMAIL](#welcome_email) | false | Send welcome email on sign up | -| [LOG_MODE](#log_mode) | console | Either "console", "file", default is "console" Use comma to separate multiple modes, e.g. "console, file" | -| [BUFFER_LENGTH](#buffer_length) | 10000 | Buffer length of channel, keep it as it is if you don't know what it is. | -| [LOG_LEVEL](#log_level) | Info | Either "Trace", "Debug", "Info", "Warn", "Error", "Critical", default is "Trace" | -| [LOG_LEVEL_CONSOLE](#log_level_console) | Info | For "console" mode only | -| [LOG_LEVEL_FILE](#log_level_file) | Info | For "file" mode only | -| [LOG_ROTATE](#log_rotate) | true | This enables automated log rotate(switch of following options), default is true | -| [LOG_MAX_LINES](#log_max_lines) | 1000000 | Max line number of single file | -| [LOG_MAX_LINES_SHIFT](#log_max_lines_shift) | 28 | Max size shift of single file, default is 28 means 1 << 28, 256MB | -| [LOG_DAILY_ROTATE](#log_daily_rotate) | true | Segment log daily, default is true | -| [LOG_MAX_DAYS](#log_max_days) | 7 | Expired days of log file(delete after max days) | -| [AMQP_PUBLISHER](#amqp_publisher) | no default | Publish Events to AMQP | -| [AMQP_PUBLISHER_HOST](#amqp_publisher_host) | no default | RabbitMQ URL | -| [AMQP_PUBLISHER_EXCHANGE](#amqp_publisher_exchange) | no default | AMQP Publisher Exchange | -| [DASHBOARD_JSON](#dashboard_json) | true | Poll a location for json files that contain dashboards | -| [DASHBOARD_JSON_PATH](#dashboard_json_path) | /usr/share/grafana/dashboards | Location to scan for json dashboards | -| [PLUGINS_PATH](#plugins_path) | /var/lib/grafana/plugins | Path to where grafana can install plugins | -| [GF_INSTALL_PLUGINS](#gf_install_plugins) | no default | Pass the plugins as a comma seperated list | - -## Development -The provided `Makefile` has various targets to help support building and publishing new images into a kubernetes cluster. - -### Environment variables -There are a few key environment variables you should be aware of when interacting with the `make` targets. - -* `BUILD_TAG` - The tag provided to the podman image when it is built (defaults to the git-sha) -* `SHORT_NAME` - The name of the image (defaults to `grafana`) -* `DRYCC_REGISTRY` - This is the registry you are using (default `registry.drycc.cc`) -* `IMAGE_PREFIX` - This is the account for the registry you are using (default `drycc`) - -### Make targets - -* `make build` - Build container image -* `make push` - Push container image to a registry -* `make upgrade` - Replaces the running grafana instance with a new one - -The typical workflow will look something like this - `DRYCC_REGISTRY= IMAGE_PREFIX=foouser make build push upgrade` diff --git a/grafana/rootfs/Dockerfile b/rootfs/Dockerfile similarity index 100% rename from grafana/rootfs/Dockerfile rename to rootfs/Dockerfile diff --git a/grafana/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json b/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json similarity index 93% rename from grafana/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json rename to rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json index 4faee35..0602bed 100644 --- a/grafana/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json @@ -1516,7 +1516,7 @@ "alias": "$tag_pod_name", "datasource": null, "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-monitor-grafana.*$\"}[1m])) by (pod) * 1000", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-grafana.*$\"}[1m])) by (pod) * 1000", "groupBy": [ { "params": [ @@ -1627,7 +1627,7 @@ "alias": "$tag_pod_name", "datasource": null, "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-monitor-grafana.*$\"}) by (pod)", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-grafana.*$\"}) by (pod)", "groupBy": [ { "params": [ @@ -3594,243 +3594,6 @@ ], "title": "REGISTRY-PROXY", "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 46, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 2, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 24 - }, - "hiddenSeries": false, - "id": 21, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-monitor-telegraf.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "pod_name" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 24 - }, - "hiddenSeries": false, - "id": 22, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-monitor-telegraf.*$\"}) by (pod)", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "pod_name" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "TELEGRAF", - "type": "row" } ], "refresh": "5s", diff --git a/grafana/rootfs/usr/share/grafana/api/dashboards/drycc_rabbitmq.json b/rootfs/usr/share/grafana/api/dashboards/drycc_rabbitmq.json similarity index 100% rename from grafana/rootfs/usr/share/grafana/api/dashboards/drycc_rabbitmq.json rename to rootfs/usr/share/grafana/api/dashboards/drycc_rabbitmq.json diff --git a/grafana/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json b/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json similarity index 100% rename from grafana/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json rename to rootfs/usr/share/grafana/api/dashboards/drycc_storage.json diff --git a/grafana/rootfs/usr/share/grafana/api/dashboards/kubernetes_deployment_metrics.json b/rootfs/usr/share/grafana/api/dashboards/kubernetes_deployment_metrics.json similarity index 100% rename from grafana/rootfs/usr/share/grafana/api/dashboards/kubernetes_deployment_metrics.json rename to rootfs/usr/share/grafana/api/dashboards/kubernetes_deployment_metrics.json diff --git a/grafana/rootfs/usr/share/grafana/api/dashboards/node.json b/rootfs/usr/share/grafana/api/dashboards/node.json similarity index 100% rename from grafana/rootfs/usr/share/grafana/api/dashboards/node.json rename to rootfs/usr/share/grafana/api/dashboards/node.json diff --git a/grafana/rootfs/usr/share/grafana/api/dashboards/pd.json b/rootfs/usr/share/grafana/api/dashboards/pd.json similarity index 100% rename from grafana/rootfs/usr/share/grafana/api/dashboards/pd.json rename to rootfs/usr/share/grafana/api/dashboards/pd.json diff --git a/grafana/rootfs/usr/share/grafana/api/dashboards/tikv_details.json b/rootfs/usr/share/grafana/api/dashboards/tikv_details.json similarity index 100% rename from grafana/rootfs/usr/share/grafana/api/dashboards/tikv_details.json rename to rootfs/usr/share/grafana/api/dashboards/tikv_details.json diff --git a/grafana/rootfs/usr/share/grafana/api/dashboards/tikv_summary.json b/rootfs/usr/share/grafana/api/dashboards/tikv_summary.json similarity index 100% rename from grafana/rootfs/usr/share/grafana/api/dashboards/tikv_summary.json rename to rootfs/usr/share/grafana/api/dashboards/tikv_summary.json diff --git a/grafana/rootfs/usr/share/grafana/api/dashboards/tikv_trouble_shooting.json b/rootfs/usr/share/grafana/api/dashboards/tikv_trouble_shooting.json similarity index 100% rename from grafana/rootfs/usr/share/grafana/api/dashboards/tikv_trouble_shooting.json rename to rootfs/usr/share/grafana/api/dashboards/tikv_trouble_shooting.json diff --git a/grafana/rootfs/usr/share/grafana/entrypoint.sh b/rootfs/usr/share/grafana/entrypoint.sh similarity index 100% rename from grafana/rootfs/usr/share/grafana/entrypoint.sh rename to rootfs/usr/share/grafana/entrypoint.sh diff --git a/grafana/rootfs/usr/share/grafana/grafana.ini.tpl b/rootfs/usr/share/grafana/grafana.ini.tpl similarity index 100% rename from grafana/rootfs/usr/share/grafana/grafana.ini.tpl rename to rootfs/usr/share/grafana/grafana.ini.tpl diff --git a/grafana/rootfs/usr/share/grafana/ldap.toml.tpl b/rootfs/usr/share/grafana/ldap.toml.tpl similarity index 100% rename from grafana/rootfs/usr/share/grafana/ldap.toml.tpl rename to rootfs/usr/share/grafana/ldap.toml.tpl diff --git a/grafana/rootfs/usr/share/grafana/start-grafana b/rootfs/usr/share/grafana/start-grafana similarity index 100% rename from grafana/rootfs/usr/share/grafana/start-grafana rename to rootfs/usr/share/grafana/start-grafana diff --git a/telegraf/Makefile b/telegraf/Makefile deleted file mode 100644 index c58adc3..0000000 --- a/telegraf/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -SHORT_NAME ?= telegraf -SHELL_SCRIPTS = rootfs/home/telegraf/start-telegraf - -include ../default.mk diff --git a/telegraf/README.md b/telegraf/README.md deleted file mode 100644 index 28ad704..0000000 --- a/telegraf/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# [Telegraf](https://influxdata.com/time-series-platform/telegraf/) - -## Description -Telegraf is a metrics collection daemon from InfluxData. It contains numerous input and output plugins that allows the user to customize what data they collect and where it is sent. This image is based on the official [drycc base image](https://github.com/drycc/base). - -## Configuration -Telegraf configuration is based largely on a toml file that is passed in when the binary starts. The issue with doing this in a containerized environment is how can you "dynamically" build this file based on values passed into the container at runtime. Therefore, this image relies on a project called [envtpl](https://github.com/arschles/envtpl) to produce the telegraf configuration file. It can take environment variables and through using go templates produce the necessary stanzas in the toml file to start telegraf. Currently, the go template only supports basic if checks and outputting values that have been set. - - ## Environment Variables - The configuration is driven via environment variables which are published to the `config.toml` file passed to telegraf when it starts. The following table gives the environment variable name and the default value if it is not set. - - | Name | Default | Description | - |-----------|---------------|---------------| - | AGENT_INTERVAL | 60s | Default data collection interval for all inputs | - | AGENT_ROUND_INTERVAL | true | Rounds collection interval to 'interval' ie, if interval="10s" then always collect on :00, :10, :20, etc. | - | AGENT_BATCH_SIZE | 3000 | This controls the size of writes that Telegraf sends to output plugins. | - | AGENT_BUFFER_LIMIT | 30000 | Telegraf will cache metric_buffer_limit metrics for each output, and will flush this buffer on a successful write. | - | AGENT_COLLECTION_JITTER | 60s | Collection jitter is used to jitter the collection by a random amount. Each plugin will sleep for a random time within jitter before collecting. This can be used to avoid many plugins querying things like sysfs at the same time, which can have a measurable effect on the system. | - | AGENT_FLUSH_INTERVAL | 60s | Default data flushing interval for all outputs. You should not set this below interval. Maximum flush_interval will be flush_interval + flush_jitter | - | AGENT_FLUSH_JITTER | 60s | Jitter the flush interval by a random amount. This is primarily to avoid large write spikes for users running a large number of telegraf instances. ie, a jitter of 5s and flush_interval 10s means flushes will happen every 10-15s. | - | AGENT_DEBUG | false | Run telegraf in debug mode. | - | AGENT_QUIET | false | Run telegraf in quiet mode. | - | AGENT_HOSTNAME | NodeName | Override default hostname | - - -## Development -The provided `Makefile` has various targets to help support building and publishing new images into a kubernetes cluster. - -### Environment variables -There are a few key environment variables you should be aware of when interacting with the `make` targets. - -* `BUILD_TAG` - The tag provided to the container image when it is built (defaults to the git-sha) -* `SHORT_NAME` - The name of the image (defaults to `grafana`) -* `DRYCC_REGISTRY` - This is the registry you are using (default `dockerhub`) -* `IMAGE_PREFIX` - This is the account for the registry you are using (default `drycc`) - -### Make targets - -* `make build` - Build container image -* `make push` - Push container image to a registry -* `make upgrade` - Replaces the running grafana instance with a new one - -The typical workflow will look something like this - `DRYCC_REGISTRY= IMAGE_PREFIX=foouser make build push upgrade`` diff --git a/telegraf/rootfs/Dockerfile b/telegraf/rootfs/Dockerfile deleted file mode 100644 index 0c743f7..0000000 --- a/telegraf/rootfs/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -ARG CODENAME -FROM registry.drycc.cc/drycc/base:${CODENAME} - -ENV DRYCC_UID=1001 \ - DRYCC_GID=1001 \ - DRYCC_HOME_DIR=/home/telegraf \ - MC_VERSION="2024.10.08.09.37.26" \ - TELEGRAF_VERSION="1.32.2" \ - ENVTPL_VERSION="1.0.0" \ - JQ_VERSION="1.7.1" - -RUN groupadd drycc --gid ${DRYCC_GID} \ - && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} - -COPY . / - -RUN install-stack mc ${MC_VERSION} \ - && install-stack telegraf ${TELEGRAF_VERSION} \ - && install-stack envtpl ${ENVTPL_VERSION} \ - && install-stack jq ${JQ_VERSION} \ - && rm -rf \ - /usr/share/doc \ - /usr/share/man \ - /usr/share/info \ - /usr/share/locale \ - /var/lib/apt/lists/* \ - /var/log/* \ - /var/cache/debconf/* \ - /etc/systemd \ - /lib/lsb \ - /lib/udev \ - /usr/lib/`echo $(uname -m)`-linux-gnu/gconv/IBM* \ - /usr/lib/`echo $(uname -m)`-linux-gnu/gconv/EBC* \ - && mkdir -p /usr/share/man/man{1..8} \ - && chown -R ${DRYCC_UID}:${DRYCC_GID} ${DRYCC_HOME_DIR} - -USER ${DRYCC_UID} -WORKDIR ${DRYCC_HOME_DIR} -CMD ["/home/telegraf/start-telegraf"] diff --git a/telegraf/rootfs/home/telegraf/config.toml.tpl b/telegraf/rootfs/home/telegraf/config.toml.tpl deleted file mode 100644 index d5a8f2c..0000000 --- a/telegraf/rootfs/home/telegraf/config.toml.tpl +++ /dev/null @@ -1,114 +0,0 @@ -# Set Tag Configuration -[tags] - -{{- if .GLOBAL_TAGS }} -[global_tags] - {{ range $index, $item := split "," .GLOBAL_TAGS }} - {{ $value := split ":" $item }}{{ $value._0 }}={{ $value._1 | quote }} - {{ end }} -{{ end }} - -# Set Agent Configuration -[agent] - interval = {{ default "60s" .AGENT_INTERVAL | quote }} - round_interval = {{ default true .AGENT_ROUND_INTERVAL }} - metric_batch_size = {{ default "3000" .AGENT_BATCH_SIZE }} - metric_buffer_limit = {{ default "30000" .AGENT_BUFFER_LIMIT }} - collection_jitter = {{ default "60s" .AGENT_COLLECTION_JITTER | quote }} - flush_interval = {{ default "60s" .AGENT_FLUSH_INTERVAL | quote }} - flush_jitter = {{ default "60s" .AGENT_FLUSH_JITTER | quote }} - debug = {{ default false .AGENT_DEBUG }} - quiet = {{ default false .AGENT_QUIET }} - flush_buffer_when_full = {{ default true .AGENT_FLUSH_BUFFER }} - {{ if .AGENT_HOSTNAME }}hostname = {{ .AGENT_HOSTNAME | quote }} {{ end }} - -# Set output configuration -{{- if .AMON_INSTANCE }} -[[outputs.amon]] - server_key = {{ .AMON_SERVER_KEY | quote }} - amon_instance = {{ .AMON_INSTANCE | quote }} - timeout = {{ default "5s" .AMON_TIMEOUT | quote }} -{{ end }} - -{{- if .AMQP_URL }} -[[outputs.amqp]] - url = {{ .AMQP_URL | quote }} - exchange = {{ default "telegraf" .AMQP_EXCHANGE | quote }} - routing_tag = {{ default "host" .AMQP_ROUTING_TAG | quote }} - {{- if .AMQP_SSL_CA }} ssl_ca = {{ .AMQP_SSL_CA | quote }} {{ end }} - {{- if .AMQP_SSL_CERT }} ssl_cert = {{ .AMQP_SSL_CERT | quote }} {{ end }} - {{- if .AMQP_SSL_KEY }} ssl_key = {{ .AMQP_SSL_KEY | quote }} {{ end }} - retention_policy = {{ default "default" .AMQP_RETENTION_POLICY | quote }} - database = = {{ default "telegraf" .AMQP_DATABASE | quote }} - precision = {{ default "s" .AMQP_PRECISION | quote }} -{{ end }} - -{{- if .DATADOG_API_KEY }} -[[outputs.datadog]] - apikey = {{ .DATADOG_API_KEY | quote }} -{{ end }} - -{{- if .GRAPHITE_SERVERS }} -[[outputs.graphite]] - servers = [{{ .GRAPHITE_SERVERS }}] - prefix = {{ default "" .GRAPHITE_PREFIX | quote }} - timeout = {{ default 2 .GRAPHITE_TIMEOUT }} -{{ end }} - -{{- if .POSTGRESQL_CONNECTION}} -[[outputs.postgresql]] - startup_error_behavior = "retry" - connection = "{{ .POSTGRESQL_CONNECTION }}" - tags_as_foreign_keys = true - create_templates = [ - '''{{`CREATE TABLE {{ .table }} ({{ .columns }})`}}''', - '''{{`SELECT create_distributed_hypertable({{ .table|quoteLiteral }}, 'time', partitioning_column => 'tag_id', number_partitions => (SELECT count(*) FROM timescaledb_information.data_nodes)::integer, replication_factor => 2, chunk_time_interval => INTERVAL '7d')`}}''', - '''{{`ALTER TABLE {{ .table }} SET (timescaledb.compress, timescaledb.compress_segmentby = 'tag_id')`}}''', - '''{{`SELECT add_retention_policy({{ .table|quoteLiteral }}, INTERVAL '1 months')`}}''', - '''{{`SELECT add_compression_policy({{ .table|quoteLiteral }}, INTERVAL '2 hours')`}}''', - ] -{{ end }} - -{{- if .KAFKA_BROKERS}} -[[outputs.kafka]] - startup_error_behavior = "retry" - brokers = [{{ .KAFKA_BROKERS }}] - topic = {{ default "telegraf" .KAFKA_TOPIC | quote }} - routing_tag = {{ .KAFKA_ROUTING_TAG | quote }} - {{- if .KAFKA_CERTIFICATE }} certificate = {{ .KAFKA_CERTIFICATE | quote }} {{ end }} - {{- if .KAFKA_KEY }} key = {{ .KAFKA_KEY | quote }} {{ end }} - {{- if .KAFKA_CA }} ca = {{ .KAFKA_CA | quote }} {{ end }} - {{- if .KAFKA_VERIFY_SSL }} verify_ssl = {{ .KAFKA_VERIFY_SSL }} {{ end }} -{{ end }} - -{{- if .LIBRATO_API_TOKEN }} -[[outputs.librato]] - api_user = {{ .LIBRATO_API_USER | quote }} - api_token = {{ .LIBRATO_API_TOKEN | quote }} - source_tag = {{ .LIBRATO_SOURCE_TAG | quote }} -{{ end }} - -{{- if .OPEN_TSDB_HOST }} - prefix = {{ .OPEN_TSDB_PREFIX | quote }} - host = {{ .OPEN_TSDB_HOST | quote }} - port = {{ .OPEN_TSDB_PORT }} - debug = {{ .OPEN_TSDB_DEUBG }} -{{ end }} - -{{- if .RIEMANN_URL }} -[[outputs.riemann]] - url = {{ .RIEMANN_URL | quote }} - transport = {{ .RIEMANN_TRANSPORT | quote }} -{{ end }} - -# Set Input Configuration - -{{- if and .KUBERNETES_URL .ENABLE_KUBERNETES }} -[[inputs.kubernetes]] - url = {{ .KUBERNETES_URL | quote }} - bearer_token = {{ .KUBERNETES_BEARER_TOKEN_PATH | quote }} - {{- if .KUBERNETES_SSL_CA }}tls_ca = {{ .KUBERNETES_SSL_CA | quote }} {{ end }} - {{- if .KUBERNETES_SSL_CERT }}tls_cert = {{ .KUBERNETES_SSL_CERT | quote }} {{ end }} - {{- if .KUBERNETES_SSL_KEY }}tls_key = {{ .KUBERNETES_SSL_KEY | quote }} {{ end }} - insecure_skip_verify = {{ default true .KUBERNETES_INSECURE_SKIP_VERIFY }} -{{ end }} diff --git a/telegraf/rootfs/home/telegraf/start-telegraf b/telegraf/rootfs/home/telegraf/start-telegraf deleted file mode 100755 index 018b4ba..0000000 --- a/telegraf/rootfs/home/telegraf/start-telegraf +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -e - -API_URL=${API_URL:-"https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT"} - -export BEARER_TOKEN_PATH TOKEN NODE_NAME NODE_IP AGENT_HOSTNAME -BEARER_TOKEN_PATH=/var/run/secrets/kubernetes.io/serviceaccount/token -TOKEN="$(cat $BEARER_TOKEN_PATH)" -NODE_NAME="$(curl -s "$API_URL/api/v1/namespaces/$POD_NAMESPACE/pods/$HOSTNAME" --header "Authorization: Bearer $TOKEN" --insecure | jq -r '.spec.nodeName')" -echo "Node Name set (${NODE_NAME})" -NODE_IP="$(curl -s "$API_URL/api/v1/namespaces/$POD_NAMESPACE/pods/$HOSTNAME" --header "Authorization: Bearer $TOKEN" --insecure | jq -r '.status.hostIP')" -echo "Node IP set (${NODE_IP})" -AGENT_HOSTNAME=$NODE_NAME - -if [ -n "$ENABLE_KUBERNETES" ]; then - export KUBERNETES_BEARER_TOKEN_PATH=${BEARER_TOKEN_PATH} - export KUBERNETES_URL="https://${NODE_IP}:10250" - echo "Setting KUBERNETES_URL: $KUBERNETES_URL" -fi - -echo "Building config.toml!" -envtpl config.toml.tpl | sed '/^$/d' > config.toml - -echo "Finished building toml..." -echo "###########################################" -echo "###########################################" -cat config.toml -echo "###########################################" -echo "###########################################" -telegraf -config config.toml -quiet From 9a808867d11af88dbe492314bfa664f6adf6af1b Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 19 Nov 2024 14:15:11 +0800 Subject: [PATCH 06/50] chore(grafana): use valkey replace redis --- README.md | 4 ++-- charts/grafana/templates/_helpers.tmpl | 4 ++-- charts/grafana/values.yaml | 4 ++-- .../grafana/api/dashboards/drycc_component_health.json | 6 +++--- rootfs/usr/share/grafana/grafana.ini.tpl | 3 +-- rootfs/usr/share/grafana/start-grafana | 6 +++--- 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 3d2a264..20f9695 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,8 @@ We welcome your input! If you have feedback, please [submit an issue][issues]. I | [DATABASE_PASSWORD](#database_password) | no default | Database password | | [DATABASE_SSL_MODE](#database_ssl_mode) | no default | For "postgres" only, either "disable", "require" or "verify-full" | | [DATABASE_PATH](#database_path) | no default | For "sqlite3" only, path relative to data_path setting | -| [SESSION_PROVIDER](#session_provider) | file | Either "memory", "file", "redis", "mysql", "postgres", default is "file" | -| [SESSION_PROVIDER_CONFIG](#session_provider_config) | no default | Provider config options
`memory: not have any config yet`
`file: session dir path, is relative to grafana data_path`
`redis: addr=127.0.0.1:6379,pool_size=100,db=grafana`
`mysql: user:password@tcp(127.0.0.1:3306)/database_name`
`postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable` | +| [SESSION_PROVIDER](#session_provider) | file | Either "memory", "file", "valkey", "mysql", "postgres", default is "file" | +| [SESSION_PROVIDER_CONFIG](#session_provider_config) | no default | Provider config options
`memory: not have any config yet`
`file: session dir path, is relative to grafana data_path`
`mysql: user:password@tcp(127.0.0.1:3306)/database_name`
`postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable` | | [SESSION_COOKIE_NAME](#session_cookie_name) | grafana_sess | Session cookie name | | [SESSION_LIFE_TIME](#session_life_time) | 86400 | Session life time, default is 86400 | | [GRAFANA_REPORTING](#grafana_reporting) | false | Server reporting, sends usage counters to stats.grafana.org every 24 hours. No ip addresses are being tracked, only simple counters to track running instances, dashboard and error counts. It is very helpful to us. Change this option to false to disable reporting. | diff --git a/charts/grafana/templates/_helpers.tmpl b/charts/grafana/templates/_helpers.tmpl index 3fb8956..7cb457a 100644 --- a/charts/grafana/templates/_helpers.tmpl +++ b/charts/grafana/templates/_helpers.tmpl @@ -74,8 +74,8 @@ env: {{- end }} - name: "PROMETHEUS_LOCATION" value: {{ .Values.global.prometheusLocation }} -- name: "REDIS_LOCATION" - value: {{ .Values.global.redisLocation }} +- name: "VALKEY_LOCATION" + value: {{ .Values.global.valkeyLocation }} - name: "RABBITMQ_LOCATION" value: {{ .Values.global.rabbitmqLocation }} - name: "STORAGE_LOCATION" diff --git a/charts/grafana/values.yaml b/charts/grafana/values.yaml index b22ecf8..68b70fb 100644 --- a/charts/grafana/values.yaml +++ b/charts/grafana/values.yaml @@ -56,7 +56,7 @@ passportUrl: "" passportKey: "" passportSecret: "" -redis: +valkey: replicas: 1 port: "6379" @@ -66,7 +66,7 @@ global: # Valid values are: # - on-cluster: Run component within the Kubernetes cluster # - off-cluster: component is running outside of the cluster and credentials and connection information will be provided. - redisLocation: "on-cluster" + valkeyLocation: "on-cluster" grafanaLocation: "on-cluster" storageLocation: "on-cluster" rabbitmqLocation: "on-cluster" diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json b/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json index 0602bed..3257148 100644 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json @@ -2944,7 +2944,7 @@ { "alias": "$tag_pod_name", "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-redis.*$\"}[1m])) by (pod) * 1000", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-valkey.*$\"}[1m])) by (pod) * 1000", "groupBy": [ { "params": [ @@ -3052,7 +3052,7 @@ { "alias": "$tag_pod_name", "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-redis.*$\"}) by (pod)", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-valkey.*$\"}) by (pod)", "groupBy": [ { "params": [ @@ -3118,7 +3118,7 @@ "refId": "A" } ], - "title": "REDIS", + "title": "VALKEY", "type": "row" }, { diff --git a/rootfs/usr/share/grafana/grafana.ini.tpl b/rootfs/usr/share/grafana/grafana.ini.tpl index 352459f..e8a4664 100644 --- a/rootfs/usr/share/grafana/grafana.ini.tpl +++ b/rootfs/usr/share/grafana/grafana.ini.tpl @@ -90,13 +90,12 @@ path = {{ .DATABASE_PATH }} #################################### Session #################################### [session] -# Either "memory", "file", "redis", "mysql", "postgres", default is "file" +# Either "memory", "file", "mysql", "postgres", default is "file" provider = {{ default "file" .SESSION_PROVIDER }} # Provider config options # memory: not have any config yet # file: session dir path, is relative to grafana data_path -# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana` # mysql: go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name` # postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable {{ if .SESSION_PROVIDER_CONFIG }} diff --git a/rootfs/usr/share/grafana/start-grafana b/rootfs/usr/share/grafana/start-grafana index 4f66f8c..3fe2f96 100755 --- a/rootfs/usr/share/grafana/start-grafana +++ b/rootfs/usr/share/grafana/start-grafana @@ -98,10 +98,10 @@ sed -i 's@r\[\\"namespace\\\"\] == \\"drycc\\"@r\[\\"namespace\\"\] == \\"\'"${N DCD="${DASHBOARD_LOCATION}"/drycc_component_health.json # remove off-cluster component panel -if [ "on-cluster" != "${REDIS_LOCATION}" ]; then - rm -rf "${DASHBOARD_LOCATION}"/redis.json +if [ "on-cluster" != "${VALKEY_LOCATION}" ]; then + rm -rf "${DASHBOARD_LOCATION}"/valkey.json # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "REDIS"))')" > "${DCD}" + echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "VALKEY"))')" > "${DCD}" fi if [ "on-cluster" != "${RABBITMQ_LOCATION}" ]; then # shellcheck disable=SC2005 From dcce4a58420aa95fdec5789d1acc9d80fc02e794 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 22 Nov 2024 21:58:20 +0800 Subject: [PATCH 07/50] chore(rabbitmq): remove rabbitmq --- README.md | 3 - charts/grafana/Chart.lock | 6 - charts/grafana/charts/common-1.1.3.tgz | Bin 10632 -> 0 bytes charts/grafana/templates/_helpers.tmpl | 2 - charts/grafana/values.yaml | 1 - .../dashboards/drycc_component_health.json | 237 - .../api/dashboards/drycc_rabbitmq.json | 8212 ----------------- rootfs/usr/share/grafana/grafana.ini.tpl | 8 - rootfs/usr/share/grafana/start-grafana | 4 - 9 files changed, 8473 deletions(-) delete mode 100644 charts/grafana/Chart.lock delete mode 100644 charts/grafana/charts/common-1.1.3.tgz delete mode 100644 rootfs/usr/share/grafana/api/dashboards/drycc_rabbitmq.json diff --git a/README.md b/README.md index 20f9695..215858a 100644 --- a/README.md +++ b/README.md @@ -105,9 +105,6 @@ We welcome your input! If you have feedback, please [submit an issue][issues]. I | [LOG_MAX_LINES_SHIFT](#log_max_lines_shift) | 28 | Max size shift of single file, default is 28 means 1 << 28, 256MB | | [LOG_DAILY_ROTATE](#log_daily_rotate) | true | Segment log daily, default is true | | [LOG_MAX_DAYS](#log_max_days) | 7 | Expired days of log file(delete after max days) | -| [AMQP_PUBLISHER](#amqp_publisher) | no default | Publish Events to AMQP | -| [AMQP_PUBLISHER_HOST](#amqp_publisher_host) | no default | RabbitMQ URL | -| [AMQP_PUBLISHER_EXCHANGE](#amqp_publisher_exchange) | no default | AMQP Publisher Exchange | | [DASHBOARD_JSON](#dashboard_json) | true | Poll a location for json files that contain dashboards | | [DASHBOARD_JSON_PATH](#dashboard_json_path) | /usr/share/grafana/dashboards | Location to scan for json dashboards | | [PLUGINS_PATH](#plugins_path) | /var/lib/grafana/plugins | Path to where grafana can install plugins | diff --git a/charts/grafana/Chart.lock b/charts/grafana/Chart.lock deleted file mode 100644 index 3b97f14..0000000 --- a/charts/grafana/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: common - repository: oci://registry.drycc.cc/charts - version: 1.1.3 -digest: sha256:4928b76b9f4927f43702c2617b139b2fe7298055b9b6d0c363a7a94530985595 -generated: "2024-11-18T16:21:05.150712088+08:00" diff --git a/charts/grafana/charts/common-1.1.3.tgz b/charts/grafana/charts/common-1.1.3.tgz deleted file mode 100644 index d1035f1a56b79241a8ae1ff4bfca514e8ceb60c4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10632 zcmV;3DRDc zVQyr3R8em|NM&qo0PKBvf7>{gVEwI6fn{71D-}|b&mM1kc2Z7y+FO^`PNjBc(o+MG zkc2WtumDhwI`Mw?{ox|Po3bU@?lQZ7DVYQwfQNVE0r1IuPNKot3{vhb;XM3m{m*bX z93CAU$o~$9!{UF3dq=~sMhAz((QtThFg*HdINCcn9DN0b8|X{vNf?LptKq%dDh}?6 zJTS%|5M`J|Ct$JH3lQ@uj=B8%EnWIPIFl68n~^y>0W;2Hb}|@DF`p%4&nNRiAW97s z9rflAM;zh^QFhXE0h&V`o&b*E{9k1`&-Z%(L{PwbkKkOj(Nixp48Dxf2?+6+Lb~iN zOz%eC$lLEdG`{}P9?ky?7$%5qNC33)|M2*@$p6Qq!)N~A#`EF@$T9GX7vP)&%m78d zCzzrDjF;f-d~o&_a6&=`U=)Cu5+5;!nD@}#e}3-yX#VGD9*2-4Hux1zCOEwb@Y z6fxwU2Poorf)PCdZ!X@P{q>!@=M8(W2Di7po`3=tczyyH;$VV9B$W8+Tp-ts=zA&{nS%fKSF;4(+z;1wj4xDSW zbb#}hpMG+HqqLa=c04<;-!|C^ddwZ)o{rR)28f8tw>*zCNP~OLqX4kkc~mWkDn_k!pGE=U3Wdn$guX{~t`Y7&rMYaF zy?abtI%k5}9pET#*~kp4$x~y(6M%jPo@msxsYfoqc9R;O1rUV{<)JEiSp}-9k%Ox7 z-GF;eViJ<+@;kHw>a9xI1laE+mOG<2S5@^F*e}5sQ5hzo;LT)=XoNUo9wvhsVO%(F zPTMf0ppQvl$ZQ~fM;xzCb}<7ECl9HWpq!26&)o0tUZ6>!ie zPT>!Nd5eXLU#bwR>tAHUkwT#Wr1)8MG~(zJcfd~Psj)_DFuaqF7o41C6XO6c`;e8j zFku&$sOtkS_dTsuLPAB&Qki`1(&%jvLJ}AdrGj@?wRuQJkcOgrMtWz`b)I$oL^A7c zk?3Sa=FM#~*#(;Bauk+Fb2iVO{ssDd@C8IT5(X3)4fA4J3$%!Eu#4s~Us_Y#FBSzc z#SxzX2gb3NXU4SB3HVxeDkXY1Huw0sSAq|_FZRwjmGk9L%Tlgdult5ip& zhAqyLmG2|*>=OO5?UEZ|HQmWNkCBRY{t?KiEH9oO840zhep?!Gw3@z`mh$=(Va{&; zhgXD|4gW_`gsb~1Znda-ICb|Yn|(VAsr!ql3ITOTE!SB%6`I_zVCt&T9*iDp6xWj= zh(>+&hzX*D3R;`?*^cwCT3UneZ~0Q|bNwy7)aLm=bey;DDaZeP7{f6R*SG@E7XLpw zI4Z>dkA??F&+-3lJP#KC=Q9L2q*KJfcd0lBdbPh+G~|d=r~#e4kQs(}k{Y1Xkc?rN zgmAirvRnN036liU@;ma4o6NNt#xI~LCw43Dnk5*&}%*LZdlTFE>=X=@x5_{K5-Pu z)r@ZV(jCRo)VuzZw8qwnI-ip0H!|*sG*fz;v^5qsVmDNoqU%FX0MVR8tTXRrC!tNAh*Vrbi^w1vyNjREP#8^iJnbI}L zF*;foYxrbANK17nV*HzjA=v7M|M}hxS81KKD640zZCk}(e*5MOP{c?=ebkk!=otDd z$ea6{MXJGs&j`hThr;Mx358A7*N4ip%eP$-=+j^Y{N11QV2m@eUkP!)maQz zFx>Z7MW>EM_j4LnU~aF3E8TM(O$UoTS)>KIr3!UuZLPe%>G*&s$NpU>WAQv0t4sz1 zoF-Cli&;q+3cz1cI5)v8fEn2L_P_CZZ@>*)0#1NGL;iJ5=^M!8iU^K4B|+k&03;E| zVOn6{+xLdx2J=}V3M9a2EWZXfGwjcRKsCe>126(G2xM>{hQRrJd+PweI8A(>P~`RA z&oI;7!pv?PG5`TZQ}ju$hX5Qgj$i=DL@r`X>iYn0Z^0P(FkwhR#?e#~rx^D&S^~Q) zj=-Pyy?tQ{Bq0|)01!Ah;)qV5k0eGWB#8nCAWDg#?H{t5+R8oFtA3<;?19jvp+{IG=YH|2cI3`EUJK|M_3He|vR%yW@b} z9D2YPz{wS-0u$D{jU=BYYTI1=w;cbM=v`ax|5xq%fA;qdiu->Kj`yD9|66&s8UL4U zy`NE%OlOLV71Jj1l)^ypM8d?sYAHB|jNK3#Fr6C>_zZF&KuTdSMyj0G>;25&6rHHF ziv96G>Z$ft;> zKYoDm9Iep9QQE^&SF}7(lY~<6tlSfOSn0BF#f+3u z(9fp@36i=r`_D{+AchpG?0Xj!TLUi8V$}tvrU$+k9Uv2{v`~3Cnlhn;rS1^=`?V0{ zSnhgLq=z}UA<;_?_>7Vpqi{sKCa1v#XX&r3y$eWD#7&`kW2+6N*-~;5RWgIQu&py-*ILprXF9zPw8mBFK<;&VE+?$gMx^$UbVEo}2>bV|4N6>gt!DF5g~(cbAtx zU7mn{CyC<6n34q!5Ll9g0zaV=3mN|wm|W;+svCFe8$H!#S?;CtF#^TsN5YX34rqZQ z04E$#K#?$1U>M4=zXl`>fKQPWVlZT24wr%eB7_2E|7WNm2g~DfCjAlW? zWR8FzLOf?{>God|%4diI98CzFE9&%NB-$rqxraIckf%GY1L8ApjClm-Snj)?q6kqK z4rb)WRpjU1%y5X@bY66MJu~bML;Si> zNvG`Oij@iS3z$G0KGF1vC;yb~KRk!i_3i!qs@?uOI@~`f+J8q!&*wjE<=MLZXT>jK zN@5`Z1;NQPA*##|O^)(ESq`H7^O6v5B{pmOhtH5I4IGud;S>`>q=8YVoAd3ZR-2BW zErkUoQv}lj2ljz*EGy-5qPdr@;Bwg08X0d|CmhV$_Ak^CKy% zBsKj`RBl0mUmaqX(4S_Vx5U3j6r4+PPiN>#+WJ7W#=vfX zVv2moQE&oeC?-ee6h#4|7fBeZTFeA^`VGt&XT=LFnVS8zTseAO|81{d$Fp5nHrFf5 zgtp!b7Lwf)I4yF9O(%NWt<3;7j@&fOx75NZfN?D5E%(-|WSjh7xvXwIs9ZJuCEij; z@C5AIso55ozMPLDT22Z*%{BZ5BoY4Iq@}9^f8_4vLU(e#Ssxo9;VDBdWQGnW2%Fzn zJq@&8Vvw~{e8KZ=q4J9Uw9~r&YP?sWNX+lte47&W8I*cvE6c5cPfE)OJPvjILk&D1 z`EY<|L4!!#^Dz`G;sS98dg%z489!c>mBp-X9H*$H&9r zKzGoak{90}jNI=J4nR7j{F26h|JSZu(fFpn`_a?C{+IpJmEYgE{{h{|&kH%;W~&Z( zk{G-d8(B2^O=HlEa4Q4O>l%>8VW8F%sXqn30(JLYPtNu%W zyC6d1n{WEB;`+pewHjf;Ocx?NX?OxM0WbpS6K2x()FsW>?b@Q9wD#aU;vSH%gxUqW zgn}rUj}h$)kV22@Y92V};0A{w7$d+bjF{l#KxoGRPvr4a94z5H1OXBTis(hU+yV7^ z2q08+6Gt3PQE92;tw5Z;&hdVR($8i=-$;XRAX_)SGWT81Y*E?hZzt&qsTBYLhy^S=up?OQ zoB>BorBiA{6fMECKYq3{PX($BKpgQrw0`z&5)#Od4vODRbCOic4So?MR$)Ch$2~4o zdJd<;Le^O}LJkOP?fo)FK(1#Bde+$*DCi&3>j)pd{W4{TgH+e+F1x0lhWp1_&hH_TM@72>dl#ztD|@0*iDh^Bg)-5jNnwR zNa_V`7)6AuwS6f)-+zAv7@kHb=!pt=f_-6$0Wn0OPOJbwrQ519-7Idb~8czeLau8vRH{?|u8Y%%}qXn()7 z{&)Cn|8M2lzWr|*^NhnNfHY92lvGAxH8q-&*>Ojeha0I%<+Dhd&zmTXqA-SvQeyflm{SqzPM5MTy+N9WN^?>8%!h87LEDE?LdhW3N17B{zBj zC^zngDrV%MvOP0qW$)&OTUkCEaK45fYr8?Li9_a&lL+AXJmFAy%+hcZW}80)km;ST z9*dFhF?|M*eWs64p={yj3p8+H{y3l}Mzl^DD{^7)7X$*CU3UW`F8tJxkQk5!qCf%D z;y_2#b;{UWlvIAN?kB^Mj{xQ`<*^Y<5aMeTnm~f_i7A5D)(&9y+W5`vH9*|=dVeK1 zXo2W!1qjcDd8GR#x^zRa%q~MkVP&QJ)1E4+^3c#+%*h;a7(fm+>h#Rid&E^3UZ)s} zv46d3VN@%In&iaVWZ3GmXhT9}RRQs!wqe=+Q{!2q`PUl%JsysV_TS;?Isa=b&(`z5 z-hGONFcYFP(g9MrX-BTC)QU%&$=s}nJ_<+HmF#5nk0|-m0#S+sB)FU=kq?gc-u(bZ=ZL9RQcx+(f>7NMY4z2jxpb)^;B2B7VTTy#O^3b|i1CqEGN+XN!N z0E*%e`Up7RIKcVUx%;4CinvJdu%1xS61ptUMH`~$EZxq<$Q;>G z`7COr!WJQ(V;zL%Gt9uQM%CLNuS8iH36GKA_j>0OlMAkZq#M>O+~X|3E%7e5q>0Y! z-ZtZ06f6aj44s&dDXo|?wFNri-3}q`L^=nsGrd~X0aM=;F!izYharcJ^7}(vQDK5n03DD>%Cd266(5H>P;Au8+%ucBt;erZK1X~ z5rv)rsmJ3)hD)K@ZxEQlg&4#F zQ5Y66%7o;%5B=M8HcOsDO9o84TFB+Bq*hl2f^wiwi~2_!I^}vRs#aGYE>fiW%(jv# z#-+TIY-L^5k8BIe^x3PnXv(HvZBe_U83xSot4)QX)VW5>PD1C}I)VKzGDjVUevRbT zALm)HY(ubIqh7c|n`5O6b z|1w0;;tw#6P0F~nAwl%z6yPX8pPFjzq@AwcQ_#w621Glrr3W|=3Mk?fqXiO_CD%1V zieRt=zBJ{7PCS_%m!KBXtklljVHd__&hsE}^l1gbhy3*f#q%>p6=!PfIep-SD4Oz_ z14ctRheXVw%puBq@VVcO$)lR}OB4&-9Lc^^Gx;(-E)m?!5LJs)DULA%j1xibO}tQm zT)_V#!#? z&fCr>0dDDl5>aIWarjg8Bf6;@_FJtC#5ie!Ja9gdW|X z7}e9Kr45UbuL4eb2rt!#mpPH8I+Qgm#+G%j)#@l~yHp?Vq&+)11&#yK(*(3gD7?-X z-HNKp3`H;dWC~~Es(8Ix`oR=6z0i`@DnA+mPva^rtO>@D;djc6D(d=#P7?7pXxht;hJp(&A@+oe` zt2j97edZ{9biQUE9S{(tPQBC;Gc`-p_m7cd25tJ%bZC(2SM{u0yQQ-+*OaxkXMO?W zqXQKAHXc4(x3gI12^4lmqX>>e6yy)UFg$u{^IF}u-W`)XeqnKZ@dUVPx->?yL7zGN zO`_2;-EgcPRo&rcZ80x)MIPgHr!(xzqjJl0^P%DZG+{>O zJ#AAKCjW}lbR@eWxj_`{3bpeljAsyx`hE51kI5WS?1NDs>?UzsF7-v92KTE|$IY@l z^D!<->beSOiauScbb~ix_&w$_%I19h_=|-Ld2GuUVSX}B!C_b6_VFlAiixa5j^WrW znM^`d$$HfnFqH-_5cOj+FLkd0Wc8M$obwyo1Y5NYBw>hIMvL6ao!leZ>`l< zG_#)usnQ!Q9qZ+0t&u{~s@>XdjUCdr<-Y22tr1ex@m_wK%W(1qs`PnPkXcv{ixgyu z|MQ>PciY|Q$N>Z~MGH(4R`x`o?b5O?%&tndG5dpY?~2sjM%-q!1v*|Y4VR7SM{!E` z=AMf6e@-ZzqV1jkFxorbE3N+@?me&nZ{vBW^B?V_fUXq90lDY5Qs8Nl^hJSe<7s+4 zuaBx%C(K|%{~ydf3G5U+WGw10lS?1Z$=_rv7ZYqdulpz0-whr*!3Y7wRJAgj(3Sd$ngSQ z$@1>)ZR$ojS=7bpgI&F0QQ|l&RU=Ng(FT>}T~fjF=)H1=2>5$6rO}$^nVE8RzlIU@*_l{PT2{ z3?>5>Q#>6AOfhf9=iv*9J6Eus{MCswWXS^MHXdp!UybTN< zQXk2wah)q#4nMDS)MiQ|DK8pQ%Qm=zFM2brY_RMIs`sTWrH|WODeHf=0=CWlpQF8_ z(*Dnb<7fT9jc4onUmfO!BX#$jJW7WXFhxAe_0SR0df}x=R0n&fh-;2+j)R#?kMvtv z$2BiI?w6`**iypOWc^n6S+@6{yo(keAa!(xEBX`@%7Ks|2Egs@>CWzTGDbcRC2~xp z&f2!qxEH+;;j)%F^r37zwgp@Qeq$sOQvB5UEMvHyD51^(yA^u+3m8L&jt+n;PlYe4 z!fz8C1^`71uz*yhwy1nlsJ7v?PGkWoY8K7R%$IH&WpAIF0w^mUEWgN{=s3huwJ`BR zUJ)hSUp-)}hFotugADzpR8{Imw|I(lu?^cfuZ5N!99>sp7pRp#5A-YiWBF&c2v`fH zCRLMXmbhG`(pp{3VC&2^=1HX4lj^UPSgZs%i~5Su=&TWI20bUgK2W{%AypjzY5be` zuv1v4FPIrwOQjTE3yxcn@er$BcoZ!#QXBJx=%jkBiA=!Duh~l-=La|f#^l;*PHiYG zZ0wpPlB_d%3b8(%HZ57CD;oTOZ?4rd%A)!UjsoDg^yA0io1HIg2KNq+P8@X;$1iU# ze?0&3-%h|~HhK^sj(m;+uq$kE>#DHZ+Y|9dl)1g_zXsaIh|;?+)q&nB*(_WFio`Hc zq;uKfg5aQxZGF#nEHC?(S=uOCU`isjaZ+y2s=t9!;|5V?E}@j!^zxuISGVGdR$Qy3 z9aekR84v!HqL#WR!4S4uH|$Vw!c=m zFc}05vb3=}2Tv!@cLJo!79a%UpBbVT6ix7_PGGLl(o>ZRkW8+Rxq0M|k zxHKyWRffY(aqq9ZfugvJT;^Ex%}0$8btrv)w{R&Zo)x22tL!Hz4YzhPK?U(TZ5_)3 zU*&JrI|H&5nsyl=&p3)p_vIV76`K#L&o{N^uHgix)iE|g z3w!<>gD;6>k;y*_!}&Arq+yWM@F@nt08* z*-%!Sdn4VdC%yY*7&skf!Hx~q&xjdV1MYJ{Mj`oXHc3^5xbKU1vq?bB_ewaoRe`Y} zQd>b^3-ai>&mKg}oEN#QYC7{@{z3Af|EN#1|2Kye!(e>RP+*JycYIvB|KoUn|JncB z%JXpk-w%Se-hR7w@6XWeqw)RBKx;iextHvIUY}`WGd^DhN>}puOnnUwCVX~hr>_93 z+U0}PpJ9%Cp3urwv|@*>;zdXapVhmL1vo;{iV8Ju<0T=y#&x9DCnwix1stJ%O`3@X zBhjy0>$khF&Gj!)9AaORNZoMCCCX4AsY_g(rCPT*SFqkS)->XZDPFU8oa;e;G_sR6 zq1@%5ixRr7B4srNT^|| zdg`kv-+>6TS{p!9RiBzM_D2cibkTroi{-4*80gaPI|OqK*a~=e3FLGHv*s|48N>?n ze#a0_9h7?pa17W-9=@@~=_)*yGwL=NyxAbT<%Vz{KC?B_DvVWAZ_R-@!M`iaY-VOZ z$yv?1-!h}Nl&qRD`~6+Ra4qp45>3g5<3FR};X!f#=kfmY{*UcE4;cRuwDtDef1+sm zKi3}w(uDhXfmxy`+xLtG*-dRIAXG;0Y5^f#e=Px_Jgl04P;(e3527ZF(;8GN z45~h;^wRT(MHwmA;%V@=dlhQ>^7)ri=S5IcE z#7pd6wD_VMh<>RUgx!Tl85Yx`?hg9rDTjKJxRJjdR;8XN12y-DM1ycV~yL%;756VPCv zcI-_LVug9XYwWEC%6(&R8tfyFy>*Vz)LGD*jL@vv_lTbRaTs5^vdHX0Iba!H=< z)Z9NVYNl-SanXkk@U%QF@jrRW@qIUfw&i~v9hCCFM#Ja*pIdnzF#fl^`aj>VKk%13 zm;2$vesYi-&HR!rY$(iEMCNKiK3RJ$AwC<@9W%aiow;+qmks4P+v_>otL))Rj{XA# zwH`XxDz3H~e5XNTM>)w$I#aB-}pR3x8esZ(fJw%vF!%gLJ-SugX|HOpx zDMj}T0=49SjgE`)pS|JedH>&bo`;M7TqxSwL=ea#+eZ@#%KB6v4$8{hFD7KQwVA+B zHG)@)4p|`Xc??Viq{RqPgZTG7ks*v{PEz1HOEp1r7QbTr8b!(M@6)&%B9x$3x zE>?-zdN;pZyxy;^;EgpMF;nhEKD?qaWKHBsRA|p)HhcLvLPZnB`Dj4)ssV*Y;_Ng7 zn+-2CkL)o88t!I32gt_(N-u>(qGLiM>dV4(Kbv%3nb;DBJafZY?hx*4v~`sM>4vS0K=0Bkc*Z)t<6BxqP=cr&eF8V@Un{a2ovS z+BQ}Nyyj4qHN+Ju=59f&20-_WT-lTLxWiW_*ikYGSDb5&!v zIb9HKmmtzsF0*C>-2$7(;*!ajX@xMCRozrFswM4`7OHM>1CqoxR_`WC^5*UnrsB%+ ztWUbxh+a|B6`{6(TC=>fTZ`{5@+y7XCrmE2V`?Fp8j^?$Y;My-Q@K^+$IbCyiUNH9 ziy&I^|BeoZ#rxlmhx>=m@!xGc>xuu~DFA#ah-)+3e-+(6n%J-GPJP%{e!gGCSGBO2 zU~dVT-NLe}TDOR7ZNgm+$TeCrFP3EsfaH}EkmA3i-~*zpCPbG5W>xVPs*a;BKdO&m zn;Ud@v*b5<2U1}iPE`^`zqMAm0{XpjXeKmR7jCb$tGncZ6@$gqSzzTP!2mNm__z|P z(hy=hlI)_L(Ua@t&kVgu_-yt0)Fj;A$AqDamAd&TqWRWUg&BtKlS!Dvc6|gj9lPki zXT0F1TJWhF#yveV z1@uvdoRr2>mq!{$OpU%Xre#D%E6xr2SZlLF>eYVzG^;xHoHe8VY=-=6maf7`e9Qhz zh%cGJC{y;F5#Wj^!LGI$z`vY=(LulSOjMd3_n`kucvi5>);vb2=hfHl3N}|?=|ZQ* z_7V-~dqD4~JBqz+Hb3yBu6dhd1$nnH<3oF45BD_t|F#yodlaC>|35h1KPu*b?HxYv z|J%y*aQR=_9TsJzqd!JvUtihovb<#MGZIaPLm^ytizZ)s%ddRp)cN{pGJ`ge+~H80 z$u+Hw*4x@BZLN#RS7+&|dyXq{Q#FM0oyUb~0I{X)%_ysb9PelP9&1jFB_ewrO*JSS zSr&@7g!3@B+c9(YYT_xGSy3BLscx5nDT)w<90h~!+bs39?``d?VtQ=4GXdnVx{3O{ zhYogqrmEBx(Ow!I%Q;N#)WdKzEx8LB4SVTC*ZM{JQv8bhq`R57DV3Q!<{P=aeN44u zaf(+Iw6f%U>Dpy8YuNhv%rx6&nu4oQf)3pL@Jvy80j@PZE08Qk7)uXi;;q7zdEP0; zEbn?Vd&Y5fn=6$`Z3bq=s2snK;Ae9_iVie*hrlmTp$x83ZyJ+zNQaeeqrW=_L1KwxuvagCL(mweZ zeT+;t;@9kBRN?8U##PV7%xMgG{hCCtd;~y&fB7|gSr%*Q1{1;FhP}y0))bdz&OESAmnch{cAWy%ll9%%%hJkNJ}=qCo~|4E^gQ zmR<|%z4F*i+J0RR7GkE;~`GywoIRHi=w diff --git a/charts/grafana/templates/_helpers.tmpl b/charts/grafana/templates/_helpers.tmpl index 7cb457a..8bd0bbe 100644 --- a/charts/grafana/templates/_helpers.tmpl +++ b/charts/grafana/templates/_helpers.tmpl @@ -76,8 +76,6 @@ env: value: {{ .Values.global.prometheusLocation }} - name: "VALKEY_LOCATION" value: {{ .Values.global.valkeyLocation }} -- name: "RABBITMQ_LOCATION" - value: {{ .Values.global.rabbitmqLocation }} - name: "STORAGE_LOCATION" value: {{ .Values.global.storageLocation }} - name: "DATABASE_LOCATION" diff --git a/charts/grafana/values.yaml b/charts/grafana/values.yaml index 68b70fb..88a5c58 100644 --- a/charts/grafana/values.yaml +++ b/charts/grafana/values.yaml @@ -69,7 +69,6 @@ global: valkeyLocation: "on-cluster" grafanaLocation: "on-cluster" storageLocation: "on-cluster" - rabbitmqLocation: "on-cluster" databaseLocation: "on-cluster" passportLocation: "on-cluster" registryLocation: "on-cluster" diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json b/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json index 3257148..8f58ced 100644 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json @@ -2648,243 +2648,6 @@ "title": "STORAGE-META-TIKV", "type": "row" }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 42, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 20 - }, - "hiddenSeries": false, - "id": 13, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-rabbitmq.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "pod_name" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 20 - }, - "hiddenSeries": false, - "id": 14, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-rabbitmq.*$\"}) by (pod)", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "pod_name" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "RABBITMQ", - "type": "row" - }, { "collapsed": true, "gridPos": { diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_rabbitmq.json b/rootfs/usr/share/grafana/api/dashboards/drycc_rabbitmq.json deleted file mode 100644 index 41d15e7..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_rabbitmq.json +++ /dev/null @@ -1,8212 +0,0 @@ -{ - "dashboard": { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": [], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "8.3.4" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "A new RabbitMQ Management Overview", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "iteration": 1659711638455, - "links": [ - { - "icon": "doc", - "tags": [], - "targetBlank": true, - "title": "Monitoring with Prometheus & Grafana", - "tooltip": "", - "type": "link", - "url": "https://www.rabbitmq.com/prometheus.html" - } - ], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#37872D", - "value": null - }, - { - "color": "#1F60C4", - "value": 10000 - }, - { - "color": "#C4162A", - "value": 100000 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "id": 64, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rabbitmq_queue_messages_ready * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Ready messages", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#C4162A", - "value": null - }, - { - "color": "#1F60C4", - "value": -1 - }, - { - "color": "#37872D", - "value": 50 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 0 - }, - "id": 62, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_global_messages_received_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Incoming messages / s", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#C4162A", - "value": null - }, - { - "color": "#1F60C4", - "value": 0 - }, - { - "color": "#37872D", - "value": 10 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 0 - }, - "id": 66, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rabbitmq_global_publishers * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Publishers", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#C4162A", - "value": null - }, - { - "color": "#1F60C4", - "value": 0 - }, - { - "color": "#37872D", - "value": 10 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 0 - }, - "id": 37, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rabbitmq_connections * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Connections", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#C4162A", - "value": null - }, - { - "color": "#1F60C4", - "value": 0 - }, - { - "color": "#37872D", - "value": 10 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 0 - }, - "id": 40, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rabbitmq_queues * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Queues", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#37872D", - "value": null - }, - { - "color": "#1F60C4", - "value": 100 - }, - { - "color": "#C4162A", - "value": 500 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 3 - }, - "id": 65, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rabbitmq_queue_messages_unacked * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", - "format": "time_series", - "hide": false, - "instant": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Unacknowledged messages", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#C4162A", - "value": null - }, - { - "color": "#1F60C4", - "value": -1 - }, - { - "color": "#37872D", - "value": 50 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 3 - }, - "id": 63, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_global_messages_redelivered_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_global_messages_delivered_consume_auto_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_global_messages_delivered_consume_manual_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_global_messages_delivered_get_auto_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_global_messages_delivered_get_manual_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", - "format": "time_series", - "hide": false, - "instant": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Outgoing messages / s", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#C4162A", - "value": null - }, - { - "color": "#1F60C4", - "value": 0 - }, - { - "color": "#37872D", - "value": 10 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 3 - }, - "id": 41, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rabbitmq_consumers * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Consumers", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#C4162A", - "value": null - }, - { - "color": "#1F60C4", - "value": 0 - }, - { - "color": "#37872D", - "value": 10 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 3 - }, - "id": 38, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rabbitmq_channels * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Channels", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#1F60C4", - "value": null - }, - { - "color": "#37872D", - "value": 3 - }, - { - "color": "#C4162A", - "value": 8 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 3 - }, - "id": 67, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rabbitmq_build_info * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Nodes", - "type": "stat" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 4, - "panels": [], - "title": "NODES", - "type": "row" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "erlang_version" - }, - "properties": [ - { - "id": "displayName", - "value": "Erlang/OTP" - }, - { - "id": "unit", - "value": "none" - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)" - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "rabbitmq_version" - }, - "properties": [ - { - "id": "displayName", - "value": "RabbitMQ" - }, - { - "id": "unit", - "value": "none" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)" - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "displayName", - "value": "Host" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "rabbitmq_node" - }, - "properties": [ - { - "id": "displayName", - "value": "Node name" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)" - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "job" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "rabbitmq_cluster" - }, - "properties": [ - { - "id": "displayName", - "value": "Cluster" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "prometheus_client_version" - }, - "properties": [ - { - "id": "displayName", - "value": "prometheus.erl" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "prometheus_plugin_version" - }, - "properties": [ - { - "id": "displayName", - "value": "rabbitmq_prometheus" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 69, - "links": [], - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "rabbitmq_build_info * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", - "format": "table", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "transformations": [ - { - "id": "merge", - "options": { - "reducers": [] - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "If the value is zero or less, the memory alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the memory alarm is triggered with a slight delay.\n\nThe kernel's view of the amount of memory used by the node can differ from what the node itself can observe. This means that this value can be negative for a sustained period of time.\n\nBy default nodes use resident set size (RSS) to compute how much memory they use. This strategy can be changed (see the guides below).\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Memory Alarms](https://www.rabbitmq.com/memory.html)\n* [Reasoning About Memory Use](https://www.rabbitmq.com/memory-use.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "decimals": 1, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 0 - }, - { - "color": "transparent", - "value": 536870912 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 7, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "(rabbitmq_resident_memory_limit_bytes * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) -\n(rabbitmq_process_resident_memory_bytes * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Memory available before publishers blocked", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric is reported for the partition where the RabbitMQ data directory is stored.\n\nIf the value is zero or less, the disk alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the free disk space alarm is triggered with a slight delay.\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Disk Space Alarms](https://www.rabbitmq.com/disk-alarms.html)\n* [Disk Space](https://www.rabbitmq.com/production-checklist.html#resource-limits-disk-space)\n* [Persistence Configuration](https://www.rabbitmq.com/persistence-conf.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "decimals": 1, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 1073741824 - }, - { - "color": "transparent", - "value": 5368709120 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 12, - "y": 11 - }, - "id": 8, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "rabbitmq_disk_space_available_bytes * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Disk space available before publishers blocked", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "When this value reaches zero, new connections will not be accepted and disk write operations may fail.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Open File Handles Limit](https://www.rabbitmq.com/production-checklist.html#resource-limits-file-handle-limit)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 500 - }, - { - "color": "transparent", - "value": 1000 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 11 - }, - "id": 2, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "(rabbitmq_process_max_fds * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) -\n(rabbitmq_process_open_fds * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "File descriptors available", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "When this value reaches zero, new connections will not be accepted.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Networking and RabbitMQ](https://www.rabbitmq.com/networking.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 500 - }, - { - "color": "transparent", - "value": 1000 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 15 - }, - "id": 5, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "(rabbitmq_process_max_tcp_sockets * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) -\n(rabbitmq_process_open_tcp_sockets * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "TCP sockets available", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 27, - "panels": [], - "title": "QUEUED MESSAGES", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total number of ready messages ready to be delivered to consumers.\n\nAim to keep this value as low as possible. RabbitMQ behaves best when messages are flowing through it. It's OK for publishers to occasionally outpace consumers, but the expectation is that consumers will eventually process all ready messages.\n\nIf this metric keeps increasing, your system will eventually run out of memory and/or disk space. Consider using TTL or Queue Length Limit to prevent unbounded message growth.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Consumers](https://www.rabbitmq.com/consumers.html)\n* [Queue Length Limit](https://www.rabbitmq.com/maxlength.html)\n* [Time-To-Live and Expiration](https://www.rabbitmq.com/ttl.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 20 - }, - "id": 9, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rabbitmq_queue_messages_ready * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Messages ready to be delivered to consumers", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The total number of messages that are either in-flight to consumers, currently being processed by consumers or simply waiting for the consumer acknowledgements to be processed by the queue. Until the queue processes the message acknowledgement, the message will remain unacknowledged.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 20 - }, - "id": 19, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rabbitmq_queue_messages_unacked * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Messages pending consumer acknowledgement", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 11, - "panels": [], - "title": "INCOMING MESSAGES", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The incoming message rate before any routing rules are applied.\n\nIf this value is lower than the number of messages published to queues, it may indicate that some messages are delivered to more than one queue.\n\nIf this value is higher than the number of messages published to queues, messages cannot be routed and will either be dropped or returned to publishers.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 26 - }, - "id": 13, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_global_messages_received_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Messages published / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of messages confirmed by the broker to publishers. Publishers must opt-in to receive message confirmations.\n\nIf this metric is consistently at zero it may suggest that publisher confirms are not used by clients. The safety of published messages is likely to be at risk.\n\n* [Publisher Confirms](https://www.rabbitmq.com/confirms.html#publisher-confirms)\n* [Publisher Confirms and Data Safety](https://www.rabbitmq.com/publishers.html#data-safety)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 26 - }, - "id": 18, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_global_messages_confirmed_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Messages confirmed to publishers / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of messages received from publishers and successfully routed to the master queue replicas.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Publishers](https://www.rabbitmq.com/publishers.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 31 - }, - "id": 61, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_global_messages_routed_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Messages routed to queues / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of messages received from publishers that have publisher confirms enabled and the broker has not confirmed yet.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 31 - }, - "id": 12, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_global_messages_received_confirm_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"} - \nrate(rabbitmq_global_messages_confirmed_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}\n) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Messages unconfirmed to publishers / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of messages that cannot be routed and are dropped. \n\nAny value above zero means message loss and likely suggests a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "red", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/rabbit/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C4162A", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 36 - }, - "id": 34, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_global_messages_unroutable_dropped_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Unroutable messages dropped / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of messages that cannot be routed and are returned back to publishers.\n\nSustained values above zero may indicate a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "red", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/rabbit/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C4162A", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 36 - }, - "id": 16, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_global_messages_unroutable_returned_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Unroutable messages returned to publishers / s", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 41 - }, - "id": 29, - "panels": [], - "title": "OUTGOING MESSAGES", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of messages delivered to consumers. It includes messages that have been redelivered.\n\nThis metric does not include messages that have been fetched by consumers using `basic.get` (consumed by polling).\n\n* [Consumers](https://www.rabbitmq.com/consumers.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 42 - }, - "id": 14, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(\n (rate(rabbitmq_global_messages_delivered_consume_auto_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\n (rate(rabbitmq_global_messages_delivered_consume_manual_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})\n) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Messages delivered / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of messages that have been redelivered to consumers. It includes messages that have been requeued automatically and redelivered due to channel exceptions or connection closures.\n\nHaving some redeliveries is expected, but if this metric is consistently non-zero, it is worth investigating why.\n\n* [Negative Acknowledgement and Requeuing of Deliveries](https://www.rabbitmq.com/confirms.html#consumer-nacks-requeue)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "orange", - "value": 20 - }, - { - "color": "red", - "value": 100 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 42 - }, - "id": 15, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_global_messages_redelivered_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Messages redelivered / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of message deliveries to consumers that use manual acknowledgement mode.\n\nWhen this mode is used, RabbitMQ waits for consumers to acknowledge messages before more messages can be delivered.\n\nThis is the safest way of consuming messages.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 47 - }, - "id": 20, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_global_messages_delivered_consume_manual_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Messages delivered with manual ack / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of message deliveries to consumers that use automatic acknowledgement mode.\n\nWhen this mode is used, RabbitMQ does not wait for consumers to acknowledge message deliveries.\n\nThis mode is fire-and-forget and does not offer any delivery safety guarantees. It tends to provide higher throughput and it may lead to consumer overload and higher consumer memory usage.\n\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 21, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_global_messages_delivered_consume_auto_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Messages delivered auto ack / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of message acknowledgements coming from consumers that use manual acknowledgement mode.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 52 - }, - "id": 22, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_global_messages_acknowledged_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Messages acknowledged / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of messages delivered to polling consumers that use automatic acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "red", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/rabbit/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C4162A", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 52 - }, - "id": 24, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_global_messages_delivered_get_auto_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Polling operations with auto ack / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of polling consumer operations that yield no result.\n\nAny value above zero means that RabbitMQ resources are wasted by polling consumers.\n\nCompare this metric to the other polling consumer metrics to see the inefficiency rate.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "red", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/rabbit/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C4162A", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 57 - }, - "id": 25, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_global_messages_get_empty_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Polling operations that yield no result / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of messages delivered to polling consumers that use manual acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "red", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/rabbit/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C4162A", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 57 - }, - "id": 23, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_global_messages_delivered_get_manual_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Polling operations with manual ack / s", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 62 - }, - "id": 53, - "panels": [], - "title": "QUEUES", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total number of queue masters per node. \n\nThis metric makes it easy to see sub-optimal queue distribution in a cluster.\n\n* [Queue Masters, Data Locality](https://www.rabbitmq.com/ha.html#master-migration-data-locality)\n* [Queues](https://www.rabbitmq.com/queues.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 63 - }, - "id": 57, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "rabbitmq_queues * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Total queues", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of queue declarations performed by clients.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "orange", - "value": 2 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 63 - }, - "id": 58, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_queues_declared_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Queues declared / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of new queues created (as opposed to redeclarations).\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "orange", - "value": 2 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 63 - }, - "id": 60, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_queues_created_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Queues created / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of queues deleted.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "orange", - "value": 2 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 63 - }, - "id": 59, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_queues_deleted_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Queues deleted / s", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 68 - }, - "id": 51, - "panels": [], - "title": "CHANNELS", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total number of channels on all currently opened connections.\n\nIf this metric grows monotonically it is highly likely a channel leak in one of the applications. Confirm channel leaks by using the _Channels opened_ and _Channels closed_ metrics.\n\n* [Channel Leak](https://www.rabbitmq.com/channels.html#channel-leaks)\n* [Channels](https://www.rabbitmq.com/channels.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 69 - }, - "id": 54, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "rabbitmq_channels * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Total channels", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of new channels opened by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "orange", - "value": 2 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 69 - }, - "id": 55, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_channels_opened_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Channels opened / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of channels closed by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "orange", - "value": 2 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 18, - "y": 69 - }, - "id": 56, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_channels_closed_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Channels closed / s", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 74 - }, - "id": 46, - "panels": [], - "title": "CONNECTIONS", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total number of client connections.\n\nIf this metric grows monotonically it is highly likely a connection leak in one of the applications. Confirm connection leaks by using the _Connections opened_ and _Connections closed_ metrics.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 75 - }, - "id": 47, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "rabbitmq_connections * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Total connections", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of new connections opened by clients. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "orange", - "value": 2 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 75 - }, - "id": 48, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_connections_opened_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Connections opened / s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The rate of connections closed. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connections](https://www.rabbitmq.com/connections.html)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "orange", - "value": 2 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2CC0C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#3274D9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#A352CC", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF780A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#96D98D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFEE52", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#8AB8FF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CA95E5", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFB357", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 18, - "y": 75 - }, - "id": 49, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.3.4", - "targets": [ - { - "expr": "sum(rate(rabbitmq_connections_closed_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{rabbitmq_node}}", - "refId": "A" - } - ], - "title": "Connections closed / s", - "type": "timeseries" - } - ], - "refresh": "15s", - "schemaVersion": 34, - "style": "dark", - "tags": [ - "rabbitmq-prometheus" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "datasource": "PBFA97CFB590B2093", - "hide": 2, - "includeAll": false, - "label": "datasource", - "multi": false, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(rabbitmq_identity_info, namespace)", - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "namespace", - "options": [], - "query": { - "query": "label_values(rabbitmq_identity_info, namespace)", - "refId": "Prometheus-namespace-Variable-Query" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(rabbitmq_identity_info{namespace=\"$namespace\"}, rabbitmq_cluster)", - "hide": 0, - "includeAll": false, - "label": "RabbitMQ Cluster", - "multi": false, - "name": "rabbitmq_cluster", - "options": [], - "query": { - "query": "label_values(rabbitmq_identity_info{namespace=\"$namespace\"}, rabbitmq_cluster)", - "refId": "Prometheus-rabbitmq_cluster-Variable-Query" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "15s", - "30s", - "1m", - "5m", - "10m" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "RabbitMQ-Overview", - "uid": "Kn5xm-gZk", - "version": 20220805, - "weekStart": "", - "gnetId": 10991 - }, - "overwrite": false -} \ No newline at end of file diff --git a/rootfs/usr/share/grafana/grafana.ini.tpl b/rootfs/usr/share/grafana/grafana.ini.tpl index e8a4664..4274e96 100644 --- a/rootfs/usr/share/grafana/grafana.ini.tpl +++ b/rootfs/usr/share/grafana/grafana.ini.tpl @@ -299,14 +299,6 @@ daily_rotate = {{ default "true" .LOG_DAILY_ROTATE }} # Expired days of log file(delete after max days), default is 7 max_days = {{ default "7" .LOG_MAX_DAYS }} -#################################### AMPQ Event Publisher ########################## -[event_publisher] -{{ if .AMQP_PUBLISHER }} -enabled = {{ .AMQP_PUBLISHER }} -rabbitmq_url = {{ .AMQP_PUBLISHER_HOST }} -exchange = {{ .AMQP_PUBLISHER_EXCHANGE }} -{{ end }} - ;#################################### Dashboard JSON files ########################## [dashboards.json] enabled = {{ default "true" .DASHBOARD_JSON }} diff --git a/rootfs/usr/share/grafana/start-grafana b/rootfs/usr/share/grafana/start-grafana index 3fe2f96..57a6b5d 100755 --- a/rootfs/usr/share/grafana/start-grafana +++ b/rootfs/usr/share/grafana/start-grafana @@ -103,10 +103,6 @@ if [ "on-cluster" != "${VALKEY_LOCATION}" ]; then # shellcheck disable=SC2005 echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "VALKEY"))')" > "${DCD}" fi -if [ "on-cluster" != "${RABBITMQ_LOCATION}" ]; then - # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "RABBITMQ"))')" > "${DCD}" -fi if [ "on-cluster" != "${DATABASE_LOCATION}" ]; then # shellcheck disable=SC2005 echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "DATABASE"))')" > "${DCD}" From efe82c94ca6857ce875aef201922d4f14fe81c87 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Mon, 23 Dec 2024 13:34:52 +0800 Subject: [PATCH 08/50] fix(wooddpecker): secsets are deprecated --- .woodpecker/build-linux.yml | 17 +++++++++++------ .woodpecker/chart.yaml | 14 +++++++++----- .woodpecker/manifest.yml | 13 ++++++++----- .woodpecker/test-linux.yml | 10 +++++----- 4 files changed, 33 insertions(+), 21 deletions(-) diff --git a/.woodpecker/build-linux.yml b/.woodpecker/build-linux.yml index 5ddefd6..75b553e 100644 --- a/.woodpecker/build-linux.yml +++ b/.woodpecker/build-linux.yml @@ -14,12 +14,17 @@ steps: - echo $CONTAINER_PASSWORD | podman login $DRYCC_REGISTRY --username $CONTAINER_USERNAME --password-stdin > /dev/null 2>&1 - export VERSION=$([ -z $CI_COMMIT_TAG ] && echo latest || echo $CI_COMMIT_TAG)-$(sed 's#/#-#g' <<< $CI_SYSTEM_PLATFORM) - make podman-build podman-push - secrets: - - codename - - dev_registry - - drycc_registry - - container_username - - container_password + environment: + CODENAME: + from_secret: codename + DEV_REGISTRY: + from_secret: dev_registry + DRYCC_REGISTRY: + from_secret: drycc_registry + CONTAINER_USERNAME: + from_secret: container_username + CONTAINER_PASSWORD: + from_secret: container_password when: event: - push diff --git a/.woodpecker/chart.yaml b/.woodpecker/chart.yaml index 5aa7eb3..40397e0 100644 --- a/.woodpecker/chart.yaml +++ b/.woodpecker/chart.yaml @@ -14,11 +14,15 @@ steps: - helm package -u charts/$${CI_REPO_NAME} --version $CHART_VERSION --app-version $APP_VERSION - echo $CONTAINER_PASSWORD | helm registry login $DRYCC_REGISTRY -u $CONTAINER_USERNAME --password-stdin - helm push $${CI_REPO_NAME}-$CHART_VERSION.tgz oci://$DRYCC_REGISTRY/$([ -z $CI_COMMIT_TAG ] && echo charts-testing || echo charts) - secrets: - - dev_registry - - drycc_registry - - container_username - - container_password + environment: + DEV_REGISTRY: + from_secret: dev_registry + DRYCC_REGISTRY: + from_secret: drycc_registry + CONTAINER_USERNAME: + from_secret: container_username + CONTAINER_PASSWORD: + from_secret: container_password when: event: - push diff --git a/.woodpecker/manifest.yml b/.woodpecker/manifest.yml index 052f5d8..d1f4483 100644 --- a/.woodpecker/manifest.yml +++ b/.woodpecker/manifest.yml @@ -8,8 +8,9 @@ steps: commands: - sed -i "s/{{project}}/$${CI_REPO_NAME}/g" .woodpecker/manifest.tmpl - sed -i "s/registry.drycc.cc/$${DRYCC_REGISTRY}/g" .woodpecker/manifest.tmpl - secrets: - - drycc_registry + environment: + DRYCC_REGISTRY: + from_secret: drycc_registry when: event: - tag @@ -26,9 +27,11 @@ steps: -v $(pwd):$(pwd) -w $(pwd) plugins/manifest - secrets: - - container_username - - container_password + environment: + CONTAINER_USERNAME: + from_secret: container_username + CONTAINER_PASSWORD: + from_secret: container_password when: event: - tag diff --git a/.woodpecker/test-linux.yml b/.woodpecker/test-linux.yml index cbb0cc6..97a7ee1 100644 --- a/.woodpecker/test-linux.yml +++ b/.woodpecker/test-linux.yml @@ -12,11 +12,11 @@ steps: image: bash commands: - make test - secrets: - - codename - - dev_registry - - drycc_registry - - container_pull_secrets + environment: + CODENAME: + from_secret: codename + DEV_REGISTRY: + from_secret: dev_registry when: event: - push From 1c4537fd34bf7fc13d8f918ecd2a8bf7c97e8b83 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 2 Apr 2025 09:07:34 +0800 Subject: [PATCH 09/50] chore(woodpecker): add cron event --- .woodpecker/build-linux.yml | 1 + .woodpecker/chart.yaml | 1 + .woodpecker/manifest.yml | 2 ++ .woodpecker/test-linux.yml | 1 + 4 files changed, 5 insertions(+) diff --git a/.woodpecker/build-linux.yml b/.woodpecker/build-linux.yml index 75b553e..585a87b 100644 --- a/.woodpecker/build-linux.yml +++ b/.woodpecker/build-linux.yml @@ -29,6 +29,7 @@ steps: event: - push - tag + - cron depends_on: - test-linux diff --git a/.woodpecker/chart.yaml b/.woodpecker/chart.yaml index 40397e0..cfcd3ec 100644 --- a/.woodpecker/chart.yaml +++ b/.woodpecker/chart.yaml @@ -27,6 +27,7 @@ steps: event: - push - tag + - cron depends_on: - manifest diff --git a/.woodpecker/manifest.yml b/.woodpecker/manifest.yml index d1f4483..88ded66 100644 --- a/.woodpecker/manifest.yml +++ b/.woodpecker/manifest.yml @@ -15,6 +15,7 @@ steps: event: - tag - push + - cron - name: publish-manifest image: bash @@ -36,6 +37,7 @@ steps: event: - tag - push + - cron depends_on: - build-linux diff --git a/.woodpecker/test-linux.yml b/.woodpecker/test-linux.yml index 97a7ee1..a1f9f7d 100644 --- a/.woodpecker/test-linux.yml +++ b/.woodpecker/test-linux.yml @@ -21,4 +21,5 @@ steps: event: - push - tag + - cron - pull_request From 7edee73a0b56c1c7a31f65db394a859e863f7c85 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 2 Apr 2025 09:13:35 +0800 Subject: [PATCH 10/50] chore(grafana): bump grafana version to 11.6.0 --- rootfs/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index e7dd282..a3746bd 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -4,7 +4,7 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/usr/share/grafana \ - GRAFANA_VERSION="11.1.3" \ + GRAFANA_VERSION="11.6.0" \ ENVTPL_VERSION="1.0.0" \ JQ_VERSION="1.7.1" From f7afb650e0ceb98e25a76284c94e129d4c70cd34 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 2 Apr 2025 11:13:20 +0800 Subject: [PATCH 11/50] fix(grafana): cve-2024-45337 --- rootfs/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index a3746bd..44979ed 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -5,7 +5,7 @@ ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/usr/share/grafana \ GRAFANA_VERSION="11.6.0" \ - ENVTPL_VERSION="1.0.0" \ + ENVTPL_VERSION="2.0.5" \ JQ_VERSION="1.7.1" RUN groupadd drycc --gid ${DRYCC_GID} \ From a9e07df872ebede3aa650990f8a4939e269e974d Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 2 Apr 2025 11:13:20 +0800 Subject: [PATCH 12/50] fix(grafana): cve-2024-45337 --- rootfs/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 44979ed..a3746bd 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -5,7 +5,7 @@ ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/usr/share/grafana \ GRAFANA_VERSION="11.6.0" \ - ENVTPL_VERSION="2.0.5" \ + ENVTPL_VERSION="1.0.0" \ JQ_VERSION="1.7.1" RUN groupadd drycc --gid ${DRYCC_GID} \ From 5f684586a7378df60e6bffdbe94750fe6760230b Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 7 May 2025 09:01:51 +0800 Subject: [PATCH 13/50] chore(charts): remove rbac --- charts/grafana/values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/grafana/values.yaml b/charts/grafana/values.yaml index 88a5c58..42e2676 100644 --- a/charts/grafana/values.yaml +++ b/charts/grafana/values.yaml @@ -73,8 +73,6 @@ global: passportLocation: "on-cluster" registryLocation: "on-cluster" prometheusLocation: "on-cluster" - # Role-Based Access Control for Kubernetes >= 1.5 - rbac: false ingressClass: "" # A domain name consists of one or more parts. # Periods (.) are used to separate these parts. From 1e072ac8bbdd33f931310650f9dd877d75d366ee Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 7 May 2025 18:04:25 +0800 Subject: [PATCH 14/50] chore(charts): use condition --- charts/grafana/templates/_helpers.tmpl | 25 +- .../grafana/templates/grafana-deployment.yaml | 2 - charts/grafana/templates/grafana-svc.yaml | 2 - charts/grafana/values.yaml | 34 +- .../dashboards/drycc_component_health.json | 484 +- .../grafana/api/dashboards/drycc_storage.json | 5437 +- .../usr/share/grafana/api/dashboards/pd.json | 11741 ----- .../grafana/api/dashboards/tikv_details.json | 41412 ---------------- .../grafana/api/dashboards/tikv_summary.json | 4926 -- .../api/dashboards/tikv_trouble_shooting.json | 7985 --- rootfs/usr/share/grafana/start-grafana | 8 +- 11 files changed, 3635 insertions(+), 68421 deletions(-) delete mode 100644 rootfs/usr/share/grafana/api/dashboards/pd.json delete mode 100644 rootfs/usr/share/grafana/api/dashboards/tikv_details.json delete mode 100644 rootfs/usr/share/grafana/api/dashboards/tikv_summary.json delete mode 100644 rootfs/usr/share/grafana/api/dashboards/tikv_trouble_shooting.json diff --git a/charts/grafana/templates/_helpers.tmpl b/charts/grafana/templates/_helpers.tmpl index 8bd0bbe..0fd355a 100644 --- a/charts/grafana/templates/_helpers.tmpl +++ b/charts/grafana/templates/_helpers.tmpl @@ -5,13 +5,10 @@ env: valueFrom: fieldRef: fieldPath: metadata.namespace -{{- if eq .Values.global.prometheusLocation "off-cluster" }} +{{- if .Values.prometheusUrl }} - name: "PROMETHEUS_URL" - valueFrom: - secretKeyRef: - name: prometheus-creds - key: url -{{- else }} + value: "{{ .Values.prometheusUrl }}" +{{- else if .Values.prometheus.enabled }} - name: "PROMETHEUS_USERNAME" valueFrom: secretKeyRef: @@ -47,7 +44,7 @@ env: {{- else }} value: http://drycc-grafana.{{ .Values.global.platformDomain }} {{- end }} -{{- if eq .Values.global.passportLocation "on-cluster"}} +{{- if .Values.passport.enabled}} - name: "DRYCC_PASSPORT_URL" {{- if .Values.global.certManagerEnabled }} value: https://drycc-passport.{{ .Values.global.platformDomain }} @@ -72,16 +69,16 @@ env: - name: DRYCC_PASSPORT_SECRET value: "{{ .Values.passportSecret }}" {{- end }} -- name: "PROMETHEUS_LOCATION" - value: {{ .Values.global.prometheusLocation }} - name: "VALKEY_LOCATION" - value: {{ .Values.global.valkeyLocation }} + value: {{ ternary "on-cluster" "off-cluster" .Values.valkey.enabled }} - name: "STORAGE_LOCATION" - value: {{ .Values.global.storageLocation }} + value: {{ ternary "on-cluster" "off-cluster" .Values.storage.enabled }} - name: "DATABASE_LOCATION" - value: {{ .Values.global.databaseLocation }} + value: {{ ternary "on-cluster" "off-cluster" .Values.database.enabled }} - name: "PASSPORT_LOCATION" - value: {{ .Values.global.passportLocation }} + value: {{ ternary "on-cluster" "off-cluster" .Values.passport.enabled }} - name: "REGISTRY_LOCATION" - value: {{ .Values.global.registryLocation }} + value: {{ ternary "on-cluster" "off-cluster" .Values.registry.enabled }} +- name: "PROMETHEUS_LOCATION" + value: {{ ternary "on-cluster" "off-cluster" .Values.prometheus.enabled }} {{- end }} diff --git a/charts/grafana/templates/grafana-deployment.yaml b/charts/grafana/templates/grafana-deployment.yaml index eb99314..4b8446e 100644 --- a/charts/grafana/templates/grafana-deployment.yaml +++ b/charts/grafana/templates/grafana-deployment.yaml @@ -1,4 +1,3 @@ -{{- if eq .Values.global.grafanaLocation "on-cluster" }} apiVersion: apps/v1 kind: Deployment metadata: @@ -61,4 +60,3 @@ spec: persistentVolumeClaim: claimName: drycc-grafana {{- end }} -{{- end }} diff --git a/charts/grafana/templates/grafana-svc.yaml b/charts/grafana/templates/grafana-svc.yaml index a91eda1..30e0854 100644 --- a/charts/grafana/templates/grafana-svc.yaml +++ b/charts/grafana/templates/grafana-svc.yaml @@ -1,4 +1,3 @@ -{{- if eq .Values.global.grafanaLocation "on-cluster" }} apiVersion: v1 kind: Service metadata: @@ -17,4 +16,3 @@ spec: targetPort: grafana selector: app: drycc-grafana -{{- end }} diff --git a/charts/grafana/values.yaml b/charts/grafana/values.yaml index 42e2676..4839b68 100644 --- a/charts/grafana/values.yaml +++ b/charts/grafana/values.yaml @@ -55,25 +55,33 @@ environment: passportUrl: "" passportKey: "" passportSecret: "" +# prometheusUrl is will no longer use the built-in prometheus component +prometheusUrl: "" valkey: + enabled: true replicas: 1 port: "6379" +grafana: + enabled: true + +storage: + enabled: true + +database: + enabled: true + +registry: + enabled: true + +passport: + enabled: true + +prometheus: + enabled: true + global: - # Set the location of Workflow's component cluster - # - # Valid values are: - # - on-cluster: Run component within the Kubernetes cluster - # - off-cluster: component is running outside of the cluster and credentials and connection information will be provided. - valkeyLocation: "on-cluster" - grafanaLocation: "on-cluster" - storageLocation: "on-cluster" - databaseLocation: "on-cluster" - passportLocation: "on-cluster" - registryLocation: "on-cluster" - prometheusLocation: "on-cluster" - ingressClass: "" # A domain name consists of one or more parts. # Periods (.) are used to separate these parts. # Each part must be 1 to 63 characters in length and can contain lowercase letters, digits, and hyphens (-). diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json b/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json index 8f58ced..453626a 100644 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json @@ -1992,7 +1992,7 @@ { "alias": "$tag_pod_name", "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage-minio.*$\"}[1m])) by (pod) * 1000", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage.*$\"}[1m])) by (pod) * 1000", "groupBy": [ { "params": [ @@ -2101,7 +2101,7 @@ { "alias": "$tag_pod_name", "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage-minio.*$\"}) by (pod)", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage.*$\"}) by (pod)", "groupBy": [ { "params": [ @@ -2167,485 +2167,7 @@ "refId": "A" } ], - "title": "STORAGE-MINIO", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 64, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 18 - }, - "hiddenSeries": false, - "id": 68, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage-meta-pd.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "pod_name" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "hide": false, - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 18 - }, - "hiddenSeries": false, - "id": 70, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage-meta-pd.*$\"}) by (pod)", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "pod_name" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "STORAGE-META-PD", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 66, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 19 - }, - "hiddenSeries": false, - "id": 72, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage-meta-tikv.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "pod_name" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "hide": false, - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 19 - }, - "hiddenSeries": false, - "id": 74, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage-meta-tikv.*$\"}) by (pod)", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - }, - { - "params": [ - "pod_name" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "STORAGE-META-TIKV", + "title": "STORAGE", "type": "row" }, { diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json b/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json index 0779c12..3efc1af 100644 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json @@ -1,1946 +1,3705 @@ { - "dashboard": { - "annotations": { - "list": [ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.3.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph (old)", + "version": "" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "MinIO Grafana Dashboard - https://min.io/", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 13502, + "graphTooltip": 0, + "id": null, + "links": [ + { + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "minio" + ], + "type": "dashboards" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dtdurations", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 1, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ { - "builtIn": 1, "datasource": { - "uid": "$DS_PROMETHEUS" + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "limit": 100, - "name": "Annotations & Alerts", - "showIn": 0, - "type": "dashboard" + "exemplar": true, + "expr": "time() - max(minio_node_process_starttime_seconds{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 } - ] + ], + "title": "Uptime", + "type": "stat" }, - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 10423, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 60, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 1 - }, - "hiddenSeries": false, - "id": 63, - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxPerRow": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"drycc\", service=~\"$service\",type=~\"$method\"}[2m])) by (code)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{code}}", - "range": true, - "refId": "A", - "step": 30 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 QPS by statusCode", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ { - "format": "short", - "logBase": 1, - "show": true + "color": "green", + "value": null } - ], - "yaxis": { - "align": false + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 65, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (instance) (minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total S3 Ingress", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false } }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 1 - }, - "hiddenSeries": false, - "id": 69, - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxPerRow": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"drycc\",service=~\"$service\"}[2m])) by (type)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "step": 30 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 QPS by method", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false + "type": "special" } - }, + ], + "unit": "bytes", + "unitScale": true + }, + "overrides": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" + "matcher": { + "id": "byName", + "options": "Free" }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 8 - }, - "hiddenSeries": false, - "id": 68, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.80, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"drycc\",service=~\"$service\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "average", - "range": true, - "refId": "A", - "step": 60 - }, + "properties": [ { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "expr": "histogram_quantile(0.80, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"drycc\",service=~\"$service\",type=~\"$method\"}[2m])) by (le, type))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B", - "step": 60 + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 Request Duration 80th percentile", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" }, - "yaxes": [ + "properties": [ { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } } - ], - "yaxis": { - "align": false + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 6, + "y": 0 + }, + "id": 50, + "interval": "1m", + "links": [], + "maxDataPoints": 100, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "topk(1, sum(minio_cluster_capacity_usable_total_bytes{job=~\"$scrape_jobs\"}) by (instance)) - topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=~\"$scrape_jobs\"}) by (instance))", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "Used", + "refId": "A", + "step": 300 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=~\"$scrape_jobs\"}) by (instance)) ", + "hide": false, + "interval": "1m", + "legendFormat": "Free", + "refId": "B" + } + ], + "title": "Capacity", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 8 - }, - "hiddenSeries": false, - "id": 67, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"drycc\",service=~\"$service\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "average", - "range": true, - "refId": "A", - "step": 60 + "color": "green", + "value": null }, { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"drycc\",service=~\"$service\", type=~\"$method\"}[2m])) by (le, type))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B", - "step": 60 + "color": "red", + "value": 80 } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 Request Duration 95th percentile", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Objects" }, - "yaxes": [ + "properties": [ { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } } - ], - "yaxis": { - "align": false - } + ] }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 8 - }, - "hiddenSeries": false, - "id": 65, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"drycc\",service=~\"$service\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "average", - "range": true, - "refId": "A", - "step": 60 - }, - { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"drycc\",service=~\"$service\", type=~\"$method\"}[2m])) by (le, type))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B", - "step": 60 + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Usage" + ], + "prefix": "All except:", + "readOnly": true } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 Request Duration 99th percentile", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, + "properties": [ { - "format": "short", - "logBase": 1, - "show": false + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" + ] } - ], - "title": "S3 api", - "type": "row" + ] }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 56, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 2 - }, - "hiddenSeries": false, - "id": 47, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket{namespace=\"drycc\",service=~\"$service\"}[2m])) by (le, exported_instance))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{exported_instance}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket{namespace=\"drycc\",service=~\"$service\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "average", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Volume Server Request Duration 99th percentile", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "gridPos": { + "h": 6, + "w": 6, + "x": 10, + "y": 0 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 2 - }, - "hiddenSeries": false, - "id": 40, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "total", - "sortDesc": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "sum(rate(SeaweedFS_volumeServer_request_total{namespace=\"drycc\",service=~\"$service\"}[2m])) by (type)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Volume Server QPS", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, + "editorMode": "code", + "exemplar": true, + "expr": "max(minio_cluster_usage_total_bytes{job=~\"$scrape_jobs\"})", + "interval": "", + "legendFormat": "Usage", + "range": true, + "refId": "A" + } + ], + "title": "Data Usage Growth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "format": "short", - "logBase": 1, - "show": true + "color": "green", + "value": null } - ], - "yaxis": { - "align": false + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 16, + "y": 0 + }, + "id": 52, + "links": [], + "options": { + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_objects_size_distribution{job=~\"$scrape_jobs\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{range}}", + "refId": "A", + "step": 300, + "useBackend": false + } + ], + "title": "Object Size Distribution", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 9 - }, - "hiddenSeries": false, - "id": 48, - "legend": { - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": true, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "sum(SeaweedFS_volumeServer_volumes{namespace=\"drycc\",service=~\"$service\"}) by (collection, type)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{collection}} {{type}}", - "range": true, - "refId": "A" + "color": "green", + "value": null }, { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "sum(max(SeaweedFS_volumeServer_max_volumes{namespace=\"drycc\",service=~\"$service\"}) by (pod))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "Total", - "refId": "B" - }, - { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "sum(max(SeaweedFS_volumeServer_read_only_volumes{namespace=\"drycc\",service=~\"$service\"}) by (pod))", - "interval": "", - "legendFormat": "Read only", - "range": true, - "refId": "C" + "color": "red", + "value": 80 } - ], - "thresholds": [], - "timeRegions": [], - "title": "Volume Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transformations": [], - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 61, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{server}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Open FDs ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ { - "format": "short", - "logBase": 1, - "show": true + "color": "green", + "value": null } - ], - "yaxis": { - "align": false + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 3 + }, + "id": 64, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (instance) (minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total S3 Egress", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 16 - }, - "hiddenSeries": false, - "id": 50, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "sum(SeaweedFS_volumeServer_total_disk_size{namespace=\"drycc\",service=~\"$service\"}) by (collection, type)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{collection}} {{type}}", - "range": true, - "refId": "A" + "color": "green", + "value": null }, { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "sum(SeaweedFS_volumeServer_total_disk_size{namespace=\"drycc\",service=~\"$service\"})", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 2, - "legendFormat": "Total", - "range": true, - "refId": "B" + "color": "red", + "value": 80 } - ], - "thresholds": [], - "timeRegions": [], - "title": "Used Disk Space by Collection and Type", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 3 + }, + "id": 62, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "minio_node_go_routine_total{job=~\"$scrape_jobs\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{server}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Goroutines", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "format": "short", - "logBase": 1, - "show": true + "color": "green", + "value": null } - ], - "yaxis": { - "align": false - } + ] }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 23 - }, - "hiddenSeries": false, - "id": 51, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "unit": "bool_on_off", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 6 + }, + "id": 94, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_health_status{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "Pool: {{pool}} Set: {{set}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Cluster Health Status", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "sum(max(SeaweedFS_volumeServer_total_disk_size{namespace=\"drycc\",service=~\"$service\"}) by (collection,pod)) by (pod)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" + "color": "green", + "value": null } - ], - "thresholds": [], - "timeRegions": [], - "title": "Used Disk Space by Host", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 6 + }, + "id": 78, + "links": [], + "maxDataPoints": 100, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(minio_cluster_drive_online_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".", + "metric": "process_start_time_seconds", + "range": false, + "refId": "A", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(minio_cluster_drive_offline_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": ".", + "range": false, + "refId": "B" + } + ], + "title": "Total Online/Offline Drives", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false + "type": "special" } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Volume Server", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 2 - }, - "id": 55, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 3 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxPerRow": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + ], + "thresholds": { + "mode": "absolute", + "steps": [ { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "rate(SeaweedFS_filer_request_total{namespace=\"drycc\",service=~\"$service\"}[2m])", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "step": 30 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Filer QPS", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ + "color": "green", + "value": null + }, { - "format": "short", - "logBase": 1, - "min": "0", - "show": true + "color": "dark-yellow", + "value": 75000000 }, { - "format": "short", - "logBase": 1, - "show": true + "color": "dark-red", + "value": 100000000 } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Filer", - "type": "row" + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 57, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 4 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket{namespace=\"drycc\",service=~\"$service\"}[2m])) by (le, type))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "range": true, - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Filer Store Request Duration 99th percentile", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 6 + }, + "id": 66, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(minio_cluster_bucket_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of Buckets", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "format": "s", - "logBase": 1, - "min": 0, - "show": true + "color": "green", + "value": null }, { - "format": "short", - "logBase": 1, - "show": true + "color": "red", + "value": 80 } - ], - "yaxis": { - "align": false + ] + }, + "unit": "binBps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 9, + "y": 6 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (rate(minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Received [{{server}}]", + "refId": "A" + } + ], + "title": "S3 API Ingress Rate ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 4 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "sum(rate(SeaweedFS_filerStore_request_total{namespace=\"drycc\",service=~\"$service\"}[2m])) by (type)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "range": true, - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Filer Store QPS", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "format": "short", - "logBase": 1, - "min": 0, - "show": true + "color": "green", + "value": null }, { - "format": "short", - "logBase": 1, - "show": true + "color": "red", + "value": 80 } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Filer Store", - "type": "row" + ] + }, + "unit": "binBps", + "unitScale": true + }, + "overrides": [] }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 58, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 5 - }, - "hiddenSeries": false, - "id": 52, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "go_memstats_alloc_bytes{namespace=~\"drycc\", service=~\"$service\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "bytes allocated", - "range": true, - "refId": "B" - }, - { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "rate(go_memstats_alloc_bytes_total{namespace=~\"drycc\", service=~\"$service\"}[30s])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "alloc rate", - "range": true, - "refId": "A" + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (rate(minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Sent [{{server}}]", + "refId": "A" + } + ], + "title": "S3 API Egress Rate ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "go_memstats_stack_inuse_bytes{namespace=~\"drycc\", service=~\"$service\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "stack inuse", - "range": true, - "refId": "C" + "color": "green", + "value": null }, { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "go_memstats_heap_inuse_bytes{namespace=~\"drycc\", service=~\"$service\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "heap inuse", - "range": true, - "refId": "D" + "color": "red", + "value": 80 } - ], - "thresholds": [], - "timeRegions": [], - "title": "Filer Go Memory Stats", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": 0, - "show": true + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 8 + }, + "id": 53, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "max(minio_cluster_nodes_online_total{job=~\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total Online Servers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } }, - { - "format": "Bps", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false + "type": "special" } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 5 - }, - "hiddenSeries": false, - "id": 54, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + ], + "thresholds": { + "mode": "absolute", + "steps": [ { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "go_gc_duration_seconds{namespace=~\"drycc\", service=~\"$service\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{quantile}}", - "range": true, - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Filer Go GC duration quantiles", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ + "color": "green", + "value": null + }, { - "format": "s", - "logBase": 1, - "min": 0, - "show": true + "color": "dark-yellow", + "value": 75000000 }, { - "format": "Bps", - "logBase": 1, - "show": true + "color": "dark-red", + "value": 100000000 } - ], - "yaxis": { - "align": false - } + ] }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 12 - }, - "hiddenSeries": false, - "id": 53, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 9 + }, + "id": 44, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(minio_cluster_usage_object_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of Objects", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "datasource": { - "uid": "$DS_PROMETHEUS" - }, - "editorMode": "code", - "expr": "go_goroutines{namespace=~\"drycc\", service=~\"$service\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "range": true, - "refId": "B" + "color": "green", + "value": null } - ], - "thresholds": [], - "timeRegions": [], - "title": "Filer Go Routines", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": 0, - "show": true - }, + ] + }, + "unit": "ns", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 10 + }, + "id": 80, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "max(minio_heal_time_last_activity_nano_seconds{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Time Since Last Heal", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "format": "short", - "logBase": 1, - "show": true + "color": "green", + "value": null } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Filer Instances", - "type": "row" - } - ], - "refresh": "30s", - "schemaVersion": 38, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, - "includeAll": false, - "label": "Datasource", - "multi": false, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": "", - "current": { - "selected": true, - "text": "All", - "value": "$__all" + ] }, + "unit": "ns", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 3, + "y": 10 + }, + "id": 81, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { "datasource": { "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "definition": "label_values({namespace=\"drycc\"},service)", - "hide": 0, - "includeAll": true, - "label": "service", - "multi": false, - "name": "service", - "options": [], - "query": { - "query": "label_values({namespace=\"drycc\"},service)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": ".*weed", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": "", - "current": { - "selected": true, - "text": "All", - "value": "$__all" + "uid": "${DS_PROMETHEUS}" }, + "exemplar": true, + "expr": "max(minio_usage_last_activity_nano_seconds{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Time Since Last Scan", + "type": "stat" + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 9, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { "datasource": { "type": "prometheus", - "uid": "$DS_PROMETHEUS" - }, - "definition": "label_values(SeaweedFS_s3_request_total{namespace=\"drycc\"},type)", - "hide": 0, - "includeAll": true, - "label": "method", - "multi": false, - "name": "method", - "options": [], - "query": { - "query": "label_values(SeaweedFS_s3_request_total{namespace=\"drycc\"},type)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "2m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] + "thresholds": [], + "timeRegions": [], + "title": "S3 API Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } }, - "timezone": "browser", - "title": "SeaweedFS", - "uid": "nh02dOVnq", - "version": 1, - "weekStart": "" + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 7, + "x": 9, + "y": 12 + }, + "hiddenSeries": false, + "id": 88, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_4xx_errors_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 API Request Error Rate (4xx)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 86, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_5xx_errors_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 API Request Error Rate (5xx)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 99, + "options": { + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_online_drives{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Online Drives", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_read_quorum{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Read Quorum", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_write_quorum{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Write Quorum", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_healing_drives{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Healing Drives", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_status{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Status", + "range": true, + "refId": "E", + "useBackend": false + } + ], + "title": "Health Breakdown", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 76, + "options": { + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "minio_node_process_resident_memory_bytes{job=~\"$scrape_jobs\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{server}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory Usage ", + "type": "bargauge" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 73, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_io_rchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Node RChar [{{server}}]", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_io_wchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Node WChar [{{server}}]", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Read, Write I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:381", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:382", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 77, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(minio_node_process_cpu_total_seconds{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{server}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of bytes received and sent on MinIO cluster", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 17, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(minio_inter_node_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Internode Bytes Received [{{server}}]", + "metric": "minio_http_requests_duration_seconds_count", + "range": true, + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_inter_node_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Internode Bytes Sent [{{server}}]", + "refId": "B" + } + ], + "title": "Internode Traffic", + "type": "timeseries" + }, + { + "aliasColors": { + "available 10.13.1.25:9000": "green", + "used 10.13.1.25:9000": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [], + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 33 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", + "interval": "", + "legendFormat": "Open FDs [{{server}}]", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Offline 10.13.1.25:9000": "dark-red", + "Total 10.13.1.25:9000": "blue" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of online drives per MinIO Server", + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 40 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_syscall_read_total{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Read Syscalls [{{server}}]", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_syscall_write_total{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Write Syscalls [{{server}}]", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Syscalls", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:185", + "decimals": 0, + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:186", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 40 + }, + "hiddenSeries": false, + "id": 95, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_scanner_objects_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Scanned Objects", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 75, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_scanner_versions_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Scanned Versions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 47 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_scanner_directories_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Scanned Directories", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "dtdurations", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 54 + }, + "id": 89, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_cluster_kms_uptime{job=~\"$scrape_jobs\"}", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "minio_cluster_kms_uptime", + "refId": "A", + "step": 60 + } + ], + "title": "KMS Uptime", + "type": "stat" + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 54 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (increase(minio_cluster_kms_request_error{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "KMS Request 4xx Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "bool_on_off", + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 54 + }, + "hiddenSeries": false, + "id": 90, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (minio_cluster_kms_online{job=~\"$scrape_jobs\"})", + "interval": "1m", + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "KMS Online(1)/Offline(0)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "bool_on_off", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 54 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_scanner_bucket_scans_finished{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Bucket Scans Finished", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 58 + }, + "hiddenSeries": false, + "id": 92, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (increase(minio_cluster_kms_request_failure{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "KMS Request 5xx Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 58 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (rate(minio_cluster_kms_request_success{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "KMS Request Success [{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "KMS Request Success Rate ", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 61 + }, + "hiddenSeries": false, + "id": 97, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_scanner_bucket_scans_started{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Bucket Scans Started", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "minio" + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(job)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "scrape_jobs", + "options": [], + "query": { + "query": "label_values(job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] }, - "overwrite": false + "timezone": "", + "title": "MinIO Dashboard", + "uid": "TgmJnqnnk", + "version": 54, + "weekStart": "" } \ No newline at end of file diff --git a/rootfs/usr/share/grafana/api/dashboards/pd.json b/rootfs/usr/share/grafana/api/dashboards/pd.json deleted file mode 100644 index 960ab1f..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/pd.json +++ /dev/null @@ -1,11741 +0,0 @@ -{ - "dashboard": { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": null, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": null, - "description": "It indicates whether the current PD is the leader or a follower.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "text": "Follower" - }, - "1": { - "text": "Leader" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "text": "Unknown" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 55, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "expr": "pd_tso_role{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", dc=\"global\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "metric": "pd_server_tso", - "refId": "A", - "step": 40 - } - ], - "title": "PD role", - "type": "stat" - }, - { - "datasource": null, - "description": "The total capacity size of the cluster", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 10, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",type=\"storage_capacity\"})", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 40 - } - ], - "title": "Storage capacity", - "type": "stat" - }, - { - "datasource": null, - "description": "The current storage size of the cluster", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 8, - "y": 0 - }, - "hideTimeOverride": false, - "id": 38, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",type=\"storage_size\"})", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 40 - } - ], - "title": "Current storage size", - "type": "stat" - }, - { - "datasource": null, - "description": "The current storage size and used ratio of the cluster", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0.01 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 0.5 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 12, - "y": 0 - }, - "hideTimeOverride": false, - "id": 37, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",type=\"storage_size\"}) / sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",type=\"storage_capacity\"})", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 40 - } - ], - "title": "Current storage used", - "type": "gauge" - }, - { - "datasource": null, - "description": "The count of healthy stores", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#d44a3a", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0 - }, - { - "color": "#299c46", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 16, - "y": 0 - }, - "id": 97, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_up_count\"})", - "format": "time_series", - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Normal stores", - "type": "stat" - }, - { - "datasource": null, - "description": "The total number of Regions without replicas", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "rgb(31, 120, 193)", - "mode": "fixed" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 20, - "y": 0 - }, - "id": 20, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",type=\"leader_count\"})", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 40 - } - ], - "title": "Number of Regions", - "type": "stat" - }, - { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": null, - "editable": true, - "error": false, - "fontSize": "100%", - "gridPos": { - "h": 7, - "w": 5, - "x": 0, - "y": 6 - }, - "hideTimeOverride": true, - "id": 96, - "links": [], - "scroll": false, - "showHeader": true, - "sort": { - "desc": false - }, - "styles": [ - { - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Metric", - "sanitize": false, - "type": "string" - }, - { - "align": "auto", - "colorMode": "cell", - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "decimals": 0, - "pattern": "Current", - "thresholds": [ - "1", - "2" - ], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "datasource": null, - "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_disconnected_count\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Disconnect Stores", - "refId": "B", - "step": 20 - }, - { - "datasource": null, - "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_unhealth_count\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Unhealth Stores", - "refId": "C", - "step": 20 - }, - { - "datasource": null, - "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_low_space_count\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "LowSpace Stores", - "refId": "D", - "step": 20 - }, - { - "datasource": null, - "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_down_count\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Down Stores", - "refId": "E", - "step": 20 - }, - { - "datasource": null, - "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_offline_count\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Offline Stores", - "refId": "F", - "step": 20 - }, - { - "datasource": null, - "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_tombstone_count\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Tombstone Stores", - "refId": "G", - "step": 20 - }, - { - "datasource": null, - "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_slow_count\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Slow Stores", - "refId": "H", - "step": 20 - } - ], - "timeFrom": "1s", - "title": "Abnormal stores", - "transform": "timeseries_aggregations", - "type": "table-old" - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 100 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "B", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0m", - "frequency": "60s", - "handler": 1, - "message": "Regions are unhealthy", - "name": "region health alert", - "noDataState": "keep_state", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "It records the unusual Regions' count which may include pending peers, down peers, extra peers, offline peers, missing peers or learner peers", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 11, - "x": 5, - "y": 6 - }, - "hiddenSeries": false, - "id": 72, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_regions_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - }, - { - "datasource": null, - "expr": "sum(pd_regions_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance, type)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B" - }, - { - "datasource": null, - "expr": "pd_regions_offline_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"offline-peer-region-count\", instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "refId": "C" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 100, - "yaxis": "left" - } - ], - "timeRegions": [], - "title": "Region health", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The current peer count of the cluster", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 6 - }, - "hiddenSeries": false, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 3, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, - "targets": [ - { - "datasource": null, - "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"region_count\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "count", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Current peer count", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "none", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 1, - "min": 0, - "noValue": "No data", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 13 - }, - "id": 1449, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": false, - "text": {} - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "avg(pd_cluster_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", action=\"removing\"} > 0)", - "interval": "", - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "Offline store progress", - "type": "gauge" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 1, - "min": 0, - "noValue": "No data", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 4, - "y": 13 - }, - "id": 1453, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": false, - "text": {} - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "avg(pd_cluster_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", action=\"preparing\"} > 0)", - "interval": "", - "legendFormat": "", - "queryType": "randomWalk", - "range": true, - "refId": "A" - } - ], - "title": "Online store progress", - "type": "gauge" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": { - "align": "left", - "displayMode": "auto", - "filterable": false, - "inspect": false - }, - "mappings": [ - { - "options": { - "1.7976931348623157e+308": { - "text": "N/A" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 8, - "y": 13 - }, - "id": 1111, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": false - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "pd_cluster_eta{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}", - "instant": true, - "interval": "", - "legendFormat": "{{action}}-{{store}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "Left time", - "transformations": [ - { - "id": "reduce", - "options": { - "reducers": [ - "last" - ] - } - } - ], - "type": "table" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": { - "align": "left", - "displayMode": "auto", - "filterable": false, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "MBs" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 12, - "y": 13 - }, - "id": 1112, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": false - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "pd_cluster_speed{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}", - "instant": true, - "interval": "", - "legendFormat": "{{action}}-{{store}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "Current scaling speed", - "transformations": [ - { - "id": "reduce", - "options": { - "reducers": [ - "last" - ] - } - } - ], - "type": "table" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 118, - "panels": [ - { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": null, - "fontSize": "90%", - "gridPos": { - "h": 7, - "w": 4, - "x": 0, - "y": 20 - }, - "hideTimeOverride": true, - "id": 116, - "links": [], - "scroll": true, - "showHeader": true, - "sort": { - "col": 1, - "desc": true - }, - "styles": [ - { - "alias": "Option", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Metric", - "preserveFormat": false, - "type": "string" - }, - { - "alias": "", - "align": "auto", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "datasource": null, - "expr": "pd_config_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "timeFrom": "1s", - "title": "PD scheduler config", - "transform": "timeseries_aggregations", - "type": "table-old" - }, - { - "columns": [], - "datasource": null, - "fontSize": "100%", - "gridPos": { - "h": 7, - "w": 5, - "x": 4, - "y": 20 - }, - "id": 1433, - "links": [], - "scroll": true, - "showHeader": true, - "sort": { - "col": 4, - "desc": true - }, - "styles": [ - { - "alias": "", - "align": "auto", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "mappingType": 1, - "pattern": "Time", - "thresholds": [], - "type": "hidden", - "unit": "short" - }, - { - "alias": "", - "align": "auto", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "__name__", - "thresholds": [], - "type": "hidden", - "unit": "short" - }, - { - "alias": "", - "align": "auto", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "instance", - "thresholds": [], - "type": "hidden", - "unit": "short" - }, - { - "alias": "", - "align": "auto", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "job", - "thresholds": [], - "type": "hidden", - "unit": "short" - }, - { - "alias": "limit (opm)", - "align": "auto", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "mappingType": 1, - "pattern": "Value", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "datasource": null, - "expr": "pd_cluster_store_limit", - "format": "table", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Store limit", - "transform": "table", - "type": "table-old" - }, - { - "columns": [], - "datasource": null, - "fontSize": "100%", - "gridPos": { - "h": 3, - "w": 5, - "x": 9, - "y": 20 - }, - "hideTimeOverride": true, - "id": 139, - "links": [], - "scroll": false, - "showHeader": true, - "sort": { - "col": 1, - "desc": false - }, - "styles": [ - { - "alias": "ID", - "align": "auto", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "mappingType": 1, - "pattern": "Metric", - "preserveFormat": false, - "sanitize": false, - "thresholds": [], - "type": "string", - "unit": "short", - "valueMaps": [ - { - "text": "", - "value": "" - } - ] - } - ], - "targets": [ - { - "datasource": null, - "expr": "pd_cluster_metadata{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "timeFrom": "1s", - "title": "Cluster ID", - "transform": "timeseries_aggregations", - "type": "table-old" - }, - { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": null, - "fontSize": "100%", - "gridPos": { - "h": 7, - "w": 5, - "x": 14, - "y": 20 - }, - "hideTimeOverride": true, - "id": 103, - "links": [], - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": false - }, - "styles": [ - { - "alias": "Type", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "type", - "type": "date" - }, - { - "alias": "Numbers", - "align": "auto", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Current", - "thresholds": [], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "align": "auto", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "datasource": null, - "expr": "pd_regions_label_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "timeFrom": "1s", - "title": "Region label isolation level", - "transform": "timeseries_aggregations", - "type": "table-old" - }, - { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": null, - "fontSize": "100%", - "gridPos": { - "h": 7, - "w": 5, - "x": 19, - "y": 20 - }, - "hideTimeOverride": true, - "id": 117, - "links": [], - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "styles": [ - { - "alias": "Label : address", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "link": false, - "linkUrl": "", - "pattern": "Metric", - "thresholds": [ - "un" - ], - "type": "string" - }, - { - "alias": "count number", - "align": "auto", - "colors": [ - "#bf1b00", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "pattern": "Current", - "thresholds": [ - "0" - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "align": "auto", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "datasource": null, - "expr": "pd_cluster_placement_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A" - } - ], - "timeFrom": "1s", - "title": "Label distribution", - "transform": "timeseries_aggregations", - "type": "table-old" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 9, - "y": 23 - }, - "hideTimeOverride": true, - "id": 115, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "/^idalloc$/", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "expr": "pd_cluster_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"idalloc\"}", - "format": "time_series", - "hide": false, - "instant": true, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "timeFrom": "1s", - "title": "Current ID allocation", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "PD cpu usage calculated with process cpu running seconds", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 27 - }, - "hiddenSeries": false, - "id": 1427, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",app_kubernetes_io_component=~\".*pd.*\"}[2m])", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{job}}-{{instance}}", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "pd_service_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",app_kubernetes_io_component=~\".*pd.*\"}", - "hide": false, - "interval": "", - "legendFormat": "{{job}}-{{instance}}-limit", - "range": true, - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU Usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "none", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "PD memory usage. ", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 27 - }, - "hiddenSeries": false, - "id": 1429, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",app_kubernetes_io_component=~\".*pd.*\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "process-{{job}}-{{instance}}", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "go_memstats_heap_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",app_kubernetes_io_component=~\".*pd.*\"}", - "hide": true, - "interval": "", - "legendFormat": "HeapSys-{{job}}-{{instance}}", - "range": true, - "refId": "B" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",app_kubernetes_io_component=~\".*pd.*\"}", - "hide": false, - "interval": "", - "legendFormat": "HeapInuse-{{job}}-{{instance}}", - "range": true, - "refId": "C" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",app_kubernetes_io_component=~\".*pd.*\"}", - "hide": true, - "interval": "", - "legendFormat": "HeapAlloc-{{job}}-{{instance}}", - "range": true, - "refId": "D" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",app_kubernetes_io_component=~\".*pd.*\"}", - "hide": true, - "interval": "", - "legendFormat": "HeapIdle-{{job}}-{{instance}}", - "range": true, - "refId": "E" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",app_kubernetes_io_component=~\".*pd.*\"}", - "hide": true, - "interval": "", - "legendFormat": "HeapReleased-{{job}}-{{instance}}", - "range": true, - "refId": "F" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",app_kubernetes_io_component=~\".*pd.*\"}", - "hide": true, - "interval": "", - "legendFormat": "GCTrigger-{{job}}-{{instance}}", - "range": true, - "refId": "G" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory Usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "none", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "PD uptime since last restart", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 34 - }, - "hiddenSeries": false, - "id": 1430, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",app_kubernetes_io_component=~\".*pd.*\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{job}}-{{instance}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Uptime", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "dtdurations", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "none", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "PD process current goroutines count", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 34 - }, - "hiddenSeries": false, - "id": 1431, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",app_kubernetes_io_component=~\".*pd.*\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{job}}-{{instance}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Goroutine Count", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "none", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 0, - "y": 41 - }, - "hideTimeOverride": true, - "id": 137, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "/^tso$/", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "expr": "pd_cluster_tso{instance=\"$instance\", type=\"tso\", dc=\"global\"}", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "title": "Current TSO Physcial", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The gap between TSO and current time", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 7, - "x": 5, - "y": 41 - }, - "hiddenSeries": false, - "id": 1443, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(pd_cluster_tso_gap_millionseconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (dc)", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{dc}}", - "refId": "A", - "step": 60 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Max TSO gap", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "dateTimeAsIso" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 0, - "y": 45 - }, - "hideTimeOverride": true, - "id": 1444, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "/^tso$/", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "expr": "pd_cluster_tso{instance=\"$instance\", type=\"tso\", dc=\"global\"}", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "title": "Current TSO Datetime", - "type": "stat" - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Cluster", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 119, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "The number of different operators that are newly created", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 21 - }, - "hiddenSeries": false, - "id": 45, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", event=\"create\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Schedule operator create", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "The number of different operators that have been checked. It mainly checks if the current step is finished; if yes, it returns the next step to be executed.", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 21 - }, - "hiddenSeries": false, - "id": 79, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", event=\"check\"}[1m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Schedule operator check", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "The number of different operators that are finished", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 28 - }, - "hiddenSeries": false, - "id": 77, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", event=\"finish\"}[1m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Schedule operator finish", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 28 - }, - "hiddenSeries": false, - "id": 78, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", event=\"timeout\"}[1m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Schedule operator timeout", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "The number of different operators that are replaced or canceled", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 35 - }, - "hiddenSeries": false, - "id": 80, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", event=\"cancel\"}[1m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", event=\"replace\"}[1m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Schedule operator replaced or canceled", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The number of operators in different status", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 35 - }, - "hiddenSeries": false, - "id": 47, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}[1m])) by (event)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{event}}", - "metric": "pd_scheduler_status", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Schedule operators count by state", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed when the operator is finished in .99", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 42 - }, - "hiddenSeries": false, - "id": 67, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (type, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}-99%", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (type, le))", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{type}}-95%", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Operator finish duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed when the operator step is finished in .99", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 42 - }, - "hiddenSeries": false, - "id": 81, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (type, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}-99%", - "refId": "A" - }, - { - "datasource": null, - "expr": "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (type, le))", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{type}}-95%", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Operator step finish duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The number of different operators that are failed to be created due to limit configuration", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 49 - }, - "hiddenSeries": false, - "id": 1426, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(pd_schedule_operator_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}[1m])) by (type,name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}-{{name}}", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(delta(pd_schedule_operator_exceeded_store_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}[1m])) by (desc)", - "hide": true, - "interval": "", - "legendFormat": "{{desc}}-exceed-store-limit", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Schedulers reach limit", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "99% operator region size", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 49 - }, - "hiddenSeries": false, - "id": 1455, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(pd_schedule_operator_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (type,le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type,le}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% operator region size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decmbytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Operator", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 120, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "The capacity size of each TiKV instance", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 22 - }, - "hiddenSeries": false, - "id": 83, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"store_capacity\"}", - "format": "time_series", - "hide": false, - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store capacity", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "logBase": 1, - "show": true - }, - { - "format": "percent", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "The available capacity size of each TiKV instance", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 22 - }, - "hiddenSeries": false, - "id": 91, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"store_available\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store available", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "The used capacity size of each TiKV instance", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 28 - }, - "hiddenSeries": false, - "id": 90, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"store_used\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store used", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 2, - "format": "decbytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "It is equal to Store available capacity size over Store capacity size for each TiKV instance", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 28 - }, - "hiddenSeries": false, - "id": 84, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"store_available\"}) by (address, store) / sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_capacity\"}) by (address, store)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store available ratio", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 2, - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "The size amplification, which is equal to Store Region size over Store used capacity size, of each TiKV instance", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 34 - }, - "hiddenSeries": false, - "id": 85, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 3, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"region_size\"}) by (address, store) / sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_used\"}) by (address, store) * 2^20", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Size amplification", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The Region score of each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 34 - }, - "hiddenSeries": false, - "id": 41, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"region_score\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "pd_scheduler_op_influence{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",scheduler=\"balance-region-scheduler\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "op-influence-{{store}}-{{type}}", - "refId": "C" - }, - { - "datasource": null, - "expr": "pd_scheduler_tolerant_resource{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",scheduler=\"balance-region-scheduler\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "tolerant-resource", - "refId": "E" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 1000000000 - } - ], - "timeRegions": [], - "title": "Store Region score", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The leader score of each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 40, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"leader_score\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 1000000000 - } - ], - "timeRegions": [], - "title": "Store leader score", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "The total Region size of each TiKV instance", - "fill": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 57, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"region_size\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store Region size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decmbytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": " \tThe total leader size of each TiKV instance", - "fill": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 46 - }, - "id": 56, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"leader_size\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store leader size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decmbytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 3, - "description": "The Region count of each TiKV instance \t", - "fill": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 46 - }, - "id": 59, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"region_count\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store Region count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 3, - "description": "The leader count of each TiKV instance", - "fill": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 52 - }, - "id": 58, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"leader_count\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store leader count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 3, - "description": "The witness count of each TiKV instance", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 52 - }, - "hiddenSeries": false, - "id": 1463, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"witness_count\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store witness count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Statistics - balance", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 135, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "description": "The total number of leader Regions under hot write on each TiKV instance", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 23 - }, - "hiddenSeries": false, - "id": 50, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"hot_write_region_as_leader\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Hot Region's leader distribution", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "description": "The total number of Regions which are not leader under hot write on each TiKV instance", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 23 - }, - "hiddenSeries": false, - "id": 51, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 1, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"hot_write_region_as_peer\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "metric": "pd_hotspot_status", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Hot Region's peer distribution", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 30 - }, - "hiddenSeries": false, - "id": 61, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_rate_bytes\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", type=\"store_regions_write_rate_bytes\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}-regions", - "refId": "B", - "step": 4 - }, - { - "datasource": null, - "exemplar": true, - "expr": "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-write-region\"}", - "hide": true, - "interval": "", - "legendFormat": "exp-byte-rate-write-leader-{{store}}", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store write rate bytes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "The total bytes of hot write on leader Regions for each TiKV instance", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 30 - }, - "hiddenSeries": false, - "id": 48, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"total_write_keys_as_leader\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "metric": "pd_hotspot_status", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "exemplar": true, - "expr": "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-write-leader\"}", - "hide": true, - "interval": "", - "legendFormat": "exp-key-rate-write-leader-{{store}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total written keys on hot leader Regions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 37 - }, - "hiddenSeries": false, - "id": 143, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_rate_keys\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", type=\"store_regions_write_rate_keys\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}-regions", - "refId": "B", - "step": 4 - }, - { - "datasource": null, - "exemplar": true, - "expr": "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-write-region\"}", - "hide": true, - "interval": "", - "legendFormat": "exp-key-rate-write-region-{{store}}", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store write rate keys", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 37 - }, - "hiddenSeries": false, - "id": 1445, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_query_rate\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "exemplar": true, - "expr": "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-write-leader\"}", - "hide": true, - "interval": "", - "legendFormat": "exp-key-rate-write-region-{{store}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store write query", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 44 - }, - "hiddenSeries": false, - "id": 146, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "pd_hotcache_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=\"total_length\", store=~\"store-$store\", type=\"write\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{store}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Hot cache write entry numbers", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 44 - }, - "hiddenSeries": false, - "id": 106, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "rate(pd_scheduler_hot_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", type=~\".*store.*\"}[1m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Selector events", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 51 - }, - "hiddenSeries": false, - "id": 150, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"out\",rw=\"write\"}[1m]))by (store)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "store-{{store}}-out", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"in\",rw=\"write\"}[1m]))by (store)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "store-{{store}}-in", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Direction of hotspot transfer leader", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 51 - }, - "hiddenSeries": false, - "id": 148, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"out\",rw=\"write\"}[1m]))by (store)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "store-{{store}}-out", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"in\",rw=\"write\"}[1m]))by (store)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "store-{{store}}-in", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Direction of hotspot move peer", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 58 - }, - "hiddenSeries": false, - "id": 1441, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": true, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_hotcache_flow_queue_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"write\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Remaining items in Queue", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Statistics - hot write", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 121, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "description": "The total number of peer Regions under hot read on each TiKV instance", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 53 - }, - "hiddenSeries": false, - "id": 60, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"hot_read_region_as_peer\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}-peer", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"hot_read_region_as_leader\"}", - "hide": true, - "interval": "", - "legendFormat": "{{address}}-store-{{store}}-leader", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Hot Region's peer distribution", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 53 - }, - "hiddenSeries": false, - "id": 105, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_rate_bytes\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "exemplar": true, - "expr": "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-read-leader\"}", - "hide": true, - "interval": "", - "legendFormat": "exp-byte-rate-read-leader-{{store}}", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-read-region\"}", - "hide": true, - "interval": "", - "legendFormat": "exp-byte-rate-read-region-{{store}}", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store read rate bytes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 60 - }, - "hiddenSeries": false, - "id": 107, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_rate_keys\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "exemplar": true, - "expr": "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-read-leader\"}", - "hide": true, - "interval": "", - "legendFormat": "exp-key-rate-read-leader-{{store}}", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-read-region\"}", - "hide": true, - "interval": "", - "legendFormat": "exp-key-rate-read-region-{{store}}", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store read rate keys", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 60 - }, - "hiddenSeries": false, - "id": 1446, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_query_rate\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "exemplar": true, - "expr": "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-read-leader\"}", - "hide": true, - "interval": "", - "legendFormat": "exp-query-rate-read-leader-{{store}}", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-read-region\"}", - "hide": true, - "interval": "", - "legendFormat": "exp-query-rate-read-region-{{store}}", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store read query", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 149, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"out\",rw=\"read\"}[1m]))by (store)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "store-{{store}}-out", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"in\",rw=\"read\"}[1m]))by (store)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "store-{{store}}-in", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Direction of hotspot transfer leader", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 144, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"out\",rw=\"read\"}[1m]))by (store)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "store-{{store}}-out", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"in\",rw=\"read\"}[1m]))by (store)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "store-{{store}}-in", - "refId": "B" - }, - { - "datasource": null, - "expr": "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"out\",rw=\"read\"}[1m]))by (store)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "store-{{store}}-out", - "refId": "C", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"in\",rw=\"read\"}[1m]))by (store)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "store-{{store}}-in", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Direction of hotspot move peer", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 74 - }, - "id": 147, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_hotcache_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=\"total_length\", store=~\"store-$store\", type=\"read\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Hot cache read entry numbers", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 74 - }, - "id": 1440, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": true, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_hotcache_flow_queue_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"read\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Remaining items in Queue", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Statistics - hot read", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 122, - "panels": [ - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The current running schedulers", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 25 - }, - "hiddenSeries": false, - "id": 46, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 1, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_scheduler_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"allow\",instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{kind}}", - "metric": "pd_scheduler_status", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler is running", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The leader movement details among TiKV instances", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 33 - }, - "hiddenSeries": false, - "id": 87, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "total", - "sortDesc": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "-sum(delta(pd_scheduler_balance_leader{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store-out\", instance=\"$instance\", type=\"move-leader\"}[1m])) by (store)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "store-{{store}}", - "refId": "A" - }, - { - "datasource": null, - "expr": "sum(delta(pd_scheduler_balance_leader{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store-in\", instance=\"$instance\", type=\"move-leader\"}[1m])) by (store)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "store-{{store}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Balance leader movement", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The Region movement details among TiKV instances", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 33 - }, - "hiddenSeries": false, - "hideTimeOverride": false, - "id": 86, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "total", - "sortDesc": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "-sum(delta(pd_scheduler_balance_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store-out\", instance=\"$instance\", type=\"move-peer\"}[1m])) by (store)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "store-{{store}}", - "refId": "A" - }, - { - "datasource": null, - "expr": "sum(delta(pd_scheduler_balance_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store-in\", instance=\"$instance\", type=\"move-peer\"}[1m])) by (store)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "store-{{store}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Balance Region movement", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The inner status of balance leader scheduler", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 41 - }, - "hiddenSeries": false, - "id": 52, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"balance-leader-scheduler\"}[5m])) by (name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "metric": "pd_scheduler_event_count", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"balance-leader-scheduler\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "total", - "metric": "pd_scheduler_event_count", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Balance leader scheduler", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The inner status of balance Region scheduler", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 41 - }, - "hiddenSeries": false, - "id": 53, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"balance-region-scheduler\"}[5m])) by (name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "metric": "pd_scheduler_event_count", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"balance-region-scheduler\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "total", - "metric": "pd_scheduler_event_count", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Balance Region scheduler", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The inner status of balance Hot Region scheduler", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 49 - }, - "hiddenSeries": false, - "id": 1458, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"balance-hot-region-scheduler\"}[5m])) by (name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "metric": "pd_scheduler_event_count", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Balance Hot Region scheduler", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The inner status of split bucket scheduler", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 49 - }, - "hiddenSeries": false, - "id": 1459, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"split-bucket-scheduler\"}[5m])) by (name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "metric": "pd_scheduler_event_count", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Split Bucket scheduler", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 57 - }, - "hiddenSeries": false, - "id": 108, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}[1m])) by (type, source, target)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{source}}-{{target}}-{{type}}", - "metric": "pd_scheduler_event_count", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Balance Direction", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 65 - }, - "id": 1424, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "expr": "pd_checker_patrol_regions_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Patrol Region time", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The replica checker's status", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 65 - }, - "hiddenSeries": false, - "id": 141, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"rule_checker\"}[1m])) by (name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Rule checker", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The replica checker's status", - "fill": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 73 - }, - "id": 70, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"replica_checker\"}[1m])) by (name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Replica checker", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The merge checker's status", - "fill": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 73 - }, - "hiddenSeries": false, - "id": 71, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"merge_checker\"}[1m])) by (name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Region merge checker", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 81 - }, - "hiddenSeries": false, - "id": 109, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", action=\"filter-source\", type!=\"store-state-tombstone-filter\"}[1m])) by (source, type, scope)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{scope}}-store-{{source}}-{{type}}", - "metric": "pd_scheduler_event_count", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Filter source", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 81 - }, - "hiddenSeries": false, - "id": 110, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", action=\"filter-target\", type!=\"store-state-tombstone-filter\"}[1m])) by (target, type, scope)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{scope}}-store-{{target}}-{{type}}", - "metric": "pd_scheduler_event_count", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", action=\"filter-target\",type=\"distinct-filter\"}[1m])) by (source, target, type, scope)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{scope}}-{{type}}-{{source}}-{{target}}", - "refId": "B" - }, - { - "datasource": null, - "expr": "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", action=\"filter-target\",type=\"rule-fit-filter\"}[1m])) by (source, target, type, scope)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{scope}}-{{type}}-{{source}}-{{target}}", - "refId": "C" - }, - { - "datasource": null, - "expr": "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", action=\"filter-target\",type=\"rule-fit-leader-filter\"}[1m])) by (source, target, type, scope)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{scope}}-{{type}}-{{source}}-{{target}}", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Filter target", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Scheduler", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 1437, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "", - "fill": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 20 - }, - "id": 1439, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"skip\"}[1m])) by (event)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "skip-{{event}}", - "refId": "A" - }, - { - "datasource": null, - "expr": "delta(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"fail\"}[1m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "fail", - "refId": "B" - }, - { - "datasource": null, - "expr": "delta(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"success\"}[1m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "success", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "scatter operator event", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 20 - }, - "id": 1435, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(pd_schedule_scatter_distribution{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",engine=\"tikv\",is_leader=\"false\"}[1m])) by (store)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "peer-{{store}}", - "refId": "A" - }, - { - "datasource": null, - "expr": "sum(delta(pd_schedule_scatter_distribution{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",engine=\"tikv\",is_leader=\"true\"}[1m])) by (store)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "leader-{{store}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "scatter store selection", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Scatter and Splitter", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 26 - }, - "id": 123, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The rate of completing each kind of gRPC commands", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 56 - }, - "hiddenSeries": false, - "id": 1, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(grpc_server_handling_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}[2m])) by (grpc_method)", - "intervalFactor": 2, - "legendFormat": "{{grpc_method}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "gRPC Completed commands rate", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 10, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed of completing each kind of gRPC commands in .99", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}[5m])) by (grpc_method, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{grpc_method}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "gRPC 99% Completed commands duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "gRPC", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 124, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The rate of completing each kind of HTTP requests", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 57 - }, - "hiddenSeries": false, - "id": 1461, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(pd_service_audit_handling_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}[2m])) by (service,ip)", - "intervalFactor": 2, - "legendFormat": "{{service}}-{{ip}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "HTTP Completed commands rate", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 10, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed of completing each kind of HTTP request in .99", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 57 - }, - "hiddenSeries": false, - "id": 1462, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(pd_service_audit_handling_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}[5m])) by (service, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{service}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "HTTP 99% Completed commands duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "HTTP", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 125, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 58 - }, - "hiddenSeries": false, - "id": 1422, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "etcd_debugging_mvcc_db_total_size_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", app_kubernetes_io_component=\"pd\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "MVCC DB total size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The rate of handling etcd transactions", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 58 - }, - "hiddenSeries": false, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(pd_txn_handle_txns_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, result)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} : {{result}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Handle transactions rate", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed of handling etcd transactions in .99", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 66 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, result, le))", - "intervalFactor": 2, - "legendFormat": "{{instance}} {{result}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% Handle transactions duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The latency of the network in .99", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 66 - }, - "hiddenSeries": false, - "id": 34, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, To, le))", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{To}}", - "metric": "", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% Peer round trip time seconds", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1 - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed of writing WAL into the persistent storage in .99", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 74 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, le))", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% WAL fsync duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 0.1 - ], - "type": "lt" - }, - "operator": { - "type": "and" - }, - "query": { - "datasourceId": 1, - "model": { - "expr": "delta(etcd_disk_wal_fsync_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"pd\"}[1m])", - "intervalFactor": 2, - "legendFormat": "{{instance}} etch disk wal fsync rate", - "refId": "A", - "step": 4 - }, - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0m", - "frequency": "60s", - "handler": 1, - "message": "PD etcd disk fsync maybe is down.", - "name": "etcd disk fsync", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The rate of writing WAL into the persistent storage", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 74 - }, - "hiddenSeries": false, - "id": 44, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "delta(etcd_disk_wal_fsync_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"pd\"}[1m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "lt", - "value": 0.1 - } - ], - "timeRegions": [], - "title": "etcd disk wal fsync rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed of committing an incremental snapshot of its most recent changes to the persistent storage in .99", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 82 - }, - "hiddenSeries": false, - "id": 151, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% backend commit duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The rate of committing an incremental snapshot of its most recent changes to the persistent storage", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 82 - }, - "hiddenSeries": false, - "id": 152, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "delta(etcd_disk_backend_commit_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "etcd disk backend commit rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The current term of Raft", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 90 - }, - "hiddenSeries": false, - "id": 92, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"term\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{job}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft term", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The last committed index of Raft", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 90 - }, - "hiddenSeries": false, - "id": 93, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"committedIndex\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{job}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft committed index", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": " \tThe last applied index of Raft", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 90 - }, - "hiddenSeries": false, - "id": 94, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"appliedIndex\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{job}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft applied index", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "etcd", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 29 - }, - "id": 126, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed of handling TiDB requests", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 59 - }, - "hiddenSeries": false, - "id": 29, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "sort": "current", - "sortDesc": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (type, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "90% tso", - "range": true, - "refId": "A", - "step": 2 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (type, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "99% tso", - "range": true, - "refId": "B", - "step": 2 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.999, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (type, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "99.9% tso", - "range": true, - "refId": "C", - "step": 2 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99999, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (type, le))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99.999% tso", - "range": true, - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "PD server TSO handle time", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed of handling TiDB requests", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 59 - }, - "hiddenSeries": false, - "id": 142, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "sort": "current", - "sortDesc": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.98, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (type, le))", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}} 98th percentile", - "range": true, - "refId": "A", - "step": 2 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "avg(rate(pd_client_request_handle_requests_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (type) / avg(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (type)", - "intervalFactor": 2, - "legendFormat": "{{type}} average", - "range": true, - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Handle requests duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The count of TSO grpc requests", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 68 - }, - "hiddenSeries": false, - "id": 28, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(pd_server_handle_tso_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (type)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "tso request/secs", - "range": true, - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Handle requests count", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The batch size of TiDB tso requests", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 68 - }, - "hiddenSeries": false, - "id": 1447, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(pd_client_request_handle_tso_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (le))", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "0.99 batch size", - "range": true, - "refId": "A" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.90, sum(rate(pd_client_request_handle_tso_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (le))", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "0.90 batch size", - "range": true, - "refId": "C" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(delta(pd_client_request_handle_tso_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) / sum(delta(pd_client_request_handle_tso_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m]))", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "avg batch size", - "range": true, - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Tso Request batch size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "TiDB", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 127, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The heartbeat update cache or kv OPS", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 60 - }, - "hiddenSeries": false, - "id": 74, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(pd_cluster_region_event{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\"}[2m])) by (address, store, event)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{event}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Heartbeat region event QPS", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "ms", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The region heartbeat handle duration in .99", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 60 - }, - "hiddenSeries": false, - "id": 140, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% Region heartbeat handle latency", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The total number of PD heartbeat messages and pending status", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 68 - }, - "hiddenSeries": false, - "id": 1460, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:363", - "alias": "/pending.*/", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster.*\"}[2m])) by (type)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "tikv_pd_pending_heartbeat_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster.*\"}", - "interval": "", - "legendFormat": "pending-{{instance}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TiKV side heartbeat statistics", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:307", - "format": "ops", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:308", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The count of the heartbeats which each TiKV instance reports to PD", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 68 - }, - "hiddenSeries": false, - "id": 54, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(delta(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"report\", status=\"ok\"}[2m])) by (address, store)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Region heartbeat report", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The count of the corresponding schedule commands which PD sends to each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 76 - }, - "hiddenSeries": false, - "id": 64, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", type=\"push\",instance=\"$instance\"}[5m])*60) by (address, store, status)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{address}}-{{status}}-store-{{store}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Region schedule push", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The count of the heartbeats with the error status", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 76 - }, - "hiddenSeries": false, - "id": 133, - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(delta(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"report\", status=\"err\"}[2m])) by (address, store)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Region heartbeat report error", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "show": true - }, - { - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The count of the heartbeats with the ok status", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 84 - }, - "hiddenSeries": false, - "id": 131, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(delta(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"report\", status=\"bind\"}[2m])) by (address, store)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Region heartbeat report active", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The store heartbeat handle duration in .99", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 84 - }, - "hiddenSeries": false, - "id": 1400, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_store_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% store heartbeat handle duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The count of the message which PD send to each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 92 - }, - "hiddenSeries": false, - "id": 1402, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(delta(pd_hbstream_region_message{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"push\", status=\"ok\"}[2m])) by (address, store)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Heartbeat stream report", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The count of the message which PD send to each TiKV instance with error status", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 92 - }, - "hiddenSeries": false, - "id": 1403, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(pd_hbstream_region_message{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"push\", status=\"err\"}[1m])) by (address, store)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Heartbeat stream report error", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The Interval of TIkv bucket report interval", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 100 - }, - "hiddenSeries": false, - "id": 1451, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(pd_server_bucket_report_interval_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster.*\", store=~\"$store\"}[2m])) by (address, store, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% Bucket Report Interval", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The State of Bucket Report", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 100 - }, - "hiddenSeries": false, - "id": 1452, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(pd_server_bucket_report{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster.*\", store=~\"$store\", instance=\"$instance\"}[2m])) by (address, store, type,status)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Bucket Report State", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Heartbeat", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 1420, - "panels": [ - { - "datasource": null, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 1407, - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(delta(pd_scheduler_read_byte_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "range": true, - "refId": "B" - } - ], - "title": "Read Region Byte", - "transparent": true, - "type": "bargauge" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 1411, - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(delta(pd_scheduler_write_byte_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "range": true, - "refId": "B" - } - ], - "title": "Write Region Byte", - "transparent": true, - "type": "bargauge" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 1406, - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(delta(pd_scheduler_read_key_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "range": true, - "refId": "C" - } - ], - "title": "Read Region Key", - "transparent": true, - "type": "bargauge" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 1412, - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(delta(pd_scheduler_write_key_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "range": true, - "refId": "C" - } - ], - "title": "Write Region Key", - "transparent": true, - "type": "bargauge" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 1408, - "interval": "", - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "9.1.1", - "repeatDirection": "h", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(delta(pd_scheduler_store_heartbeat_interval_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" - } - ], - "title": "Store Heartbeat Interval", - "transparent": true, - "type": "bargauge" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 1409, - "interval": "", - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "9.1.1", - "repeatDirection": "h", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(delta(pd_scheduler_region_heartbeat_interval_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" - } - ], - "title": "Region Heartbeat Interval", - "transparent": true, - "type": "bargauge" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 56 - }, - "id": 1454, - "interval": "", - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "9.1.1", - "repeatDirection": "h", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(delta(pd_server_bucket_report_interval_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" - } - ], - "title": "Bucket Report Interval", - "transparent": true, - "type": "bargauge" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 1455, - "interval": "", - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "9.1.1", - "repeatDirection": "h", - "targets": [ - { - "datasource": null, - "expr": "sum(delta(pd_scheduler_buckets_hot_degree_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}[1m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Hot Degree of Bucket", - "transparent": true, - "type": "bargauge" - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Heartbeat distribution ", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 128, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 33 - }, - "hiddenSeries": false, - "id": 112, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_region_syncer_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"sync_index\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Syncer index", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 33 - }, - "hiddenSeries": false, - "id": 113, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "pd_region_syncer_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"last_index\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "History last index", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Region storage", - "type": "row" - } - ], - "refresh": "30s", - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": null, - "definition": "label_values(pd_cluster_status, k8s_cluster)", - "hide": 2, - "includeAll": false, - "label": "K8s-cluster", - "multi": false, - "name": "k8s_cluster", - "options": [], - "query": { - "query": "label_values(pd_cluster_status, k8s_cluster)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": null, - "definition": "", - "hide": 2, - "includeAll": false, - "label": "tidb_cluster", - "multi": false, - "name": "tidb_cluster", - "options": [], - "query": { - "query": "label_values(pd_cluster_status{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", - "refId": "prometheus-datasource-tidb_cluster-Variable-Query" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "10.0.0.71:2379", - "value": "10.0.0.71:2379" - }, - "datasource": null, - "definition": "", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "instance", - "options": [], - "query": { - "query": "label_values(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", - "refId": "prometheus-datasource-instance-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": null, - "definition": "label_values(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, store)", - "hide": 0, - "includeAll": true, - "label": "store", - "multi": true, - "name": "store", - "options": [], - "query": { - "query": "label_values(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, store)", - "refId": "prometheus-datasource-store-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Cluster-PD", - "uid": "Q6RuHYIWk", - "version": 1, - "weekStart": "" - }, - "overwrite": false -} \ No newline at end of file diff --git a/rootfs/usr/share/grafana/api/dashboards/tikv_details.json b/rootfs/usr/share/grafana/api/dashboards/tikv_details.json deleted file mode 100644 index ecdb6bc..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/tikv_details.json +++ /dev/null @@ -1,41412 +0,0 @@ -{ - "dashboard": { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": null, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 13620, - "panels": [ - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Write Duration Composition", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 1 - }, - "hiddenSeries": false, - "id": 12842, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Write Raft Log .99", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "Propose Wait .99", - "range": true, - "refId": "B" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "Apply Wait .99", - "range": true, - "refId": "C" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "Replicate Raft Log .99", - "range": true, - "refId": "D" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "Apply Duration .99", - "range": true, - "refId": "E" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Write Pipeline Duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Read Duration Composition", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 1 - }, - "hiddenSeries": false, - "id": 12970, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Get Snapshot .99", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "Cop Wait .99", - "range": true, - "refId": "B" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "Cop Handle .99", - "range": true, - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Cop Read Duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Duration", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 2742, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The storage size per TiKV instance", - "editable": true, - "error": false, - "fill": 5, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 2 - }, - "hiddenSeries": false, - "id": 56, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"used\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The available capacity size of each TiKV instance", - "editable": true, - "error": false, - "fill": 5, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 2 - }, - "hiddenSeries": false, - "id": 1706, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"available\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Available size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The capacity size per TiKV instance", - "editable": true, - "error": false, - "fill": 5, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 2 - }, - "hiddenSeries": false, - "id": 1707, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"capacity\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Capacity size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU usage of each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "hiddenSeries": false, - "id": 1708, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", app_kubernetes_io_component=~\".*tikv\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The memory usage per TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 - }, - "hiddenSeries": false, - "id": 1709, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", app_kubernetes_io_component=~\".*tikv\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The I/O utilization per TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "hiddenSeries": false, - "id": 1710, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{device}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "IO utilization", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total bytes of read and write in each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 18 - }, - "hiddenSeries": false, - "id": 1711, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"wal_file_bytes\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}-write", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=~\"bytes_read|iter_bytes_read\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}-read", - "range": true, - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "MBps", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The QPS per command in each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 26 - }, - "hiddenSeries": false, - "id": 1713, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (instance,type)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{type}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "QPS", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total number of the gRPC message failures", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 26 - }, - "hiddenSeries": false, - "id": 1712, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}-grpc-msg-fail", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(delta(tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"noop\"}[2m])) by (instance) < 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-pd-heartbeat", - "range": true, - "refId": "B" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{type}}", - "range": true, - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Errps", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe number of leaders on each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 34 - }, - "hiddenSeries": false, - "id": 1715, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m]) < -10", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Leader", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of Regions and Buckets on each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 34 - }, - "hiddenSeries": false, - "id": 1714, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"buckets\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - buckets", - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Region", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "TiKV uptime since the last restart", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 42 - }, - "hiddenSeries": false, - "id": 4106, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", app_kubernetes_io_component=~\".*tikv\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Uptime", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "dtdurations", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Cluster", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 2 - }, - "id": 2743, - "panels": [ - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 0 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "frequency": "60s", - "handler": 1, - "name": "Critical error alert", - "noDataState": "no_data", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 3 - }, - "hiddenSeries": false, - "id": 2741, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{type}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0, - "visible": true - } - ], - "timeRegions": [], - "title": "Critical error", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "Indicates occurrences of events that make the TiKV instance unavailable temporarily, such as Write Stall, Channel Full, Scheduler Busy, and Coprocessor Full", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 10 - }, - "hiddenSeries": false, - "id": 1584, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "scheduler-{{instance}}", - "metric": "", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "channelfull-{{instance}}-{{type}}", - "metric": "", - "range": true, - "refId": "B", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type='full'}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "coprocessor-{{instance}}", - "metric": "", - "range": true, - "refId": "C", - "step": 4 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\", db=~\"$db\"}) by (instance, db)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "stall-{{instance}}-{{db}}", - "refId": "D" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_write_msg_block_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "hide": false, - "interval": "", - "legendFormat": "store-write-channelfull-{{instance}}", - "range": true, - "refId": "E" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Server is busy", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 0 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "10s", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "frequency": "10s", - "handler": 1, - "message": "TiKV server report failures", - "name": "server report failures alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total number of reporting failure messages", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 10 - }, - "hiddenSeries": false, - "id": 18, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_server_report_failure_msg_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type,instance,store_id)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{type}} - to - {{store_id}}", - "metric": "tikv_server_raft_store_msg_total", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0, - "visible": true - } - ], - "timeRegions": [], - "title": "Server report failures", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of different raftstore errors on each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 17 - }, - "hiddenSeries": false, - "id": 1718, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", status!~\"success|all\"}[2m])) by (instance, status)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{status}}", - "metric": "", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raftstore error", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of scheduler errors per type on each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 17 - }, - "hiddenSeries": false, - "id": 1719, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"snapshot_err|prepare_write_err\"}[2m])) by (instance, stage)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{stage}}", - "metric": "", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler error", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of different coprocessor errors on each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 24 - }, - "hiddenSeries": false, - "id": 1720, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{reason}}", - "metric": "", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Coprocessor error", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of gRPC message errors per type on each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 24 - }, - "hiddenSeries": false, - "id": 1721, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{type}}", - "metric": "", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "gRPC message error", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of dropped leaders per TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 31 - }, - "hiddenSeries": false, - "id": 1722, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "range": true, - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Leader drop", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of missing leaders per TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 31 - }, - "hiddenSeries": false, - "id": 1723, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_raftstore_leader_missing{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Leader missing", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "RocksDB damaged SST files", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 38 - }, - "hiddenSeries": false, - "id": 23763572510, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "tikv_rocksdb_damaged_files{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}-existed", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "tikv_rocksdb_damaged_files_deleted{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", - "hide": false, - "interval": "", - "legendFormat": "{{instance}}-deleted", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Damaged files", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "the count of Log Replication Reject caused by follower memory insufficient", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 38 - }, - "hiddenSeries": false, - "id": 23763572588, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_server_raft_append_rejects{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (instance)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}-existed", - "queryType": "randomWalk", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Log Replication Rejected", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Errors", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 2744, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The size of each column family", - "editable": true, - "error": false, - "fill": 3, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 4 - }, - "hiddenSeries": false, - "id": 33, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CF size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 0 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "datasourceId": 1, - "model": { - "expr": "sum(rate(tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{type}}", - "metric": "", - "refId": "A", - "step": 10 - }, - "params": [ - "A", - "10s", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "frequency": "10s", - "handler": 1, - "message": "TiKV channel full", - "name": "TiKV channel full alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total number of channel full errors on each TiKV instance", - "editable": true, - "error": false, - "fill": 3, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 4 - }, - "hiddenSeries": false, - "id": 22, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{type}}", - "metric": "", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0 - } - ], - "timeRegions": [], - "title": "Channel full", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of leaders being written on each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 - }, - "hiddenSeries": false, - "id": 75, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_region_written_keys_bucket", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Active written leaders", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 1073741824 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "B", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "frequency": "60s", - "handler": 1, - "name": "approximate region size alert", - "noDataState": "no_data", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The approximate Region size", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 - }, - "hiddenSeries": false, - "id": 1481, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "metric": "", - "range": true, - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_region_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_region_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Approximate Region size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 20 - }, - "hiddenSeries": false, - "id": 3638, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": false, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Approximate Region size Histogram", - "tooltip": { - "msResolution": false, - "shared": false, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "histogram", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The average rate of writing bytes to Regions per TiKV instance", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 20 - }, - "hiddenSeries": false, - "id": 58, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_region_written_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (instance) / sum(rate(tikv_region_written_bytes_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_regi", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Region average written bytes", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "timeseries", - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 28 - }, - "heatmap": {}, - "hideZeroBuckets": false, - "highlightCards": true, - "id": 3646, - "legend": { - "show": false - }, - "links": [], - "options": { - "calculate": true, - "calculation": {}, - "cellGap": 2, - "cellValues": {}, - "color": { - "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", - "scale": "exponential", - "scheme": "Oranges", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": false - }, - "rowsFrame": { - "layout": "auto" - }, - "showValue": "never", - "tooltip": { - "show": true, - "yHistogram": false - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "decbytes" - } - }, - "pluginVersion": "9.1.1", - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_region_written_bytes_bucket[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_regi", - "range": true, - "refId": "A", - "step": 10 - } - ], - "title": "Region written bytes", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "format": "decbytes", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The average rate of written keys to Regions per TiKV instance", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 28 - }, - "hiddenSeries": false, - "id": 57, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_region_written_keys_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance) / sum(rate(tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_region_written_keys_bucket", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Region average written keys", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "timeseries", - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 36 - }, - "heatmap": {}, - "hideZeroBuckets": false, - "highlightCards": true, - "id": 3647, - "legend": { - "show": false - }, - "links": [], - "options": { - "calculate": true, - "calculation": {}, - "cellGap": 2, - "cellValues": {}, - "color": { - "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", - "scale": "exponential", - "scheme": "Oranges", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": false - }, - "rowsFrame": { - "layout": "auto" - }, - "showValue": "never", - "tooltip": { - "show": true, - "yHistogram": false - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "short" - } - }, - "pluginVersion": "9.1.1", - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_region_written_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_region_written_keys_bucket", - "range": true, - "refId": "A", - "step": 10 - } - ], - "title": "Region written keys", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "format": "short", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The size of requests into request batch per TiKV instance", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 36 - }, - "hiddenSeries": false, - "id": 3720, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_server_request_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_server_request_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{type}} avg", - "range": true, - "refId": "A" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(tikv_server_request_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{type}} 99", - "range": true, - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Request batch input", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 10, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe number of peers in hibernated state", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 44 - }, - "hiddenSeries": false, - "id": 3730, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_raftstore_hibernated_peer_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, state)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{state}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Hibernate Peers", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 44 - }, - "hiddenSeries": false, - "id": 7266, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "tikv_server_mem_trace_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore-.*\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{name}}", - "refId": "A" - }, - { - "datasource": null, - "expr": "raft_engine_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-raft-engine", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory trace", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 52 - }, - "hiddenSeries": false, - "id": 9560, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "rate(tikv_raft_entries_evict_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}[2m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft Entry Cache Evicts", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 52 - }, - "hiddenSeries": false, - "id": 12971, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_server_address_resolve_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Resolve address duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 60 - }, - "hiddenSeries": false, - "id": 23763572581, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_yatp_pool_schedule_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, name))", - "hide": false, - "interval": "", - "legendFormat": "{{name}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% Thread Pool Schedule Wait Duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 2, - "max": "30", - "show": true - }, - { - "format": "short", - "logBase": 2, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 60 - }, - "hiddenSeries": false, - "id": 23763572692, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_yatp_pool_schedule_wait_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name) / sum(rate(tikv_yatp_pool_schedule_wait_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)", - "hide": false, - "interval": "", - "legendFormat": "{{name}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Average Thread Pool Schedule Wait Duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 2, - "max": "30", - "show": true - }, - { - "format": "short", - "logBase": 2, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 68 - }, - "hiddenSeries": false, - "id": 23763572784, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*/", - "stack": "A" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_storage_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", metric=\"block_read_time\"}[2m])) by (req)", - "hide": false, - "interval": "", - "legendFormat": "{{req}}", - "queryType": "randomWalk", - "range": true, - "refId": "A" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_coprocessor_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", metric=\"block_read_time\"}[2m])) by (req)", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "copr-{{req}}", - "queryType": "randomWalk", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Disk IO time per second", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 68 - }, - "hiddenSeries": false, - "id": 23763572785, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*/", - "stack": "A" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_storage_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", metric=\"block_read_byte\"}[2m])) by (req)", - "interval": "", - "legendFormat": "{{req}}", - "queryType": "randomWalk", - "range": true, - "refId": "A" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_coprocessor_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", metric=\"block_read_byte\"}[2m])) by (req)", - "hide": false, - "interval": "", - "legendFormat": "copr-{{req}}", - "queryType": "randomWalk", - "range": true, - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Disk IO bytes per second", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "binBps", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Server", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 2745, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of different kinds of gRPC message", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 5 - }, - "hiddenSeries": false, - "id": 95, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_grpc_msg_duration_seconds_bucket", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "gRPC message count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of different kinds of gRPC message which is failed", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 5 - }, - "hiddenSeries": false, - "id": 107, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_grpc_msg_fail_total", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "gRPC message failed", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The execution time of gRPC message", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 13 - }, - "hiddenSeries": false, - "id": 98, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (le, type))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% gRPC message duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 10, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 13 - }, - "hiddenSeries": false, - "id": 2532, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_grpc_msg_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Average gRPC message duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 21 - }, - "hiddenSeries": false, - "id": 2533, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(tikv_server_grpc_req_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "99% request", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(tikv_server_grpc_resp_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99% response", - "range": true, - "refId": "B" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_server_grpc_req_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_server_grpc_req_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg request", - "range": true, - "refId": "C" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_server_grpc_resp_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_server_grpc_resp_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg response", - "range": true, - "refId": "D" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(tikv_server_request_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99% kv get batch", - "range": true, - "refId": "E" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_server_request_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_server_request_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg kv batch", - "range": true, - "refId": "F" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "gRPC batch size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 21 - }, - "hiddenSeries": false, - "id": 2534, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(tikv_server_raft_message_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "99%", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_server_raft_message_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_server_raft_message_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "range": true, - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "raft message batch size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The QPS of different sources of gRPC request", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 29 - }, - "hiddenSeries": false, - "id": 23763572858, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_grpc_request_source_counter_vec{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (source)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_grpc_msg_duration_seconds_bucket", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "gRPC request sources QPS", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The duration of different sources of gRPC request", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 29 - }, - "hiddenSeries": false, - "id": 23763572859, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_grpc_request_source_duration_vec{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (source)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_grpc_msg_duration_seconds_bucket", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "gRPC request sources duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The QPS of different resource groups of gRPC request", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 37 - }, - "hiddenSeries": false, - "id": 23763573091, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_grpc_resource_group_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_grpc_msg_duration_seconds_bucket", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "gRPC resource group QPS", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "gRPC", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 2746, - "panels": [ - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 1.7 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "datasourceId": 1, - "model": { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(raftstore|rs)_.*\"}[2m])) by (instance)", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 20 - }, - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0m", - "frequency": "60s", - "handler": 1, - "message": "TiKV raftstore thread CPU usage is high", - "name": "TiKV raft store CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of raftstore thread", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 6 - }, - "hiddenSeries": false, - "id": 61, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(raftstore|rs)_.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 1.7, - "visible": true - } - ], - "timeRegions": [], - "title": "Raft store CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 1.8 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0m", - "frequency": "1m", - "handler": 1, - "message": "TiKV async apply thread CPU usage is high", - "name": "TiKV async apply CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of async apply", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 6 - }, - "hiddenSeries": false, - "id": 79, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"apply_[0-9]+\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 1.8, - "visible": true - } - ], - "timeRegions": [], - "title": "Async apply CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "alert": { - "alertRuleTags": {}, - "conditions": [ - { - "evaluator": { - "params": [ - 0.8 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0m", - "frequency": "60s", - "handler": 1, - "message": "TiKV store writer thread CPU usage is high", - "name": "Store writer CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of store writer thread", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 13 - }, - "hiddenSeries": false, - "id": 13115, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_write.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.8, - "visible": true - } - ], - "timeRegions": [], - "title": "Store writer CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 3.6 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0m", - "frequency": "1m", - "handler": 1, - "message": "TiKV gRPC poll thread CPU usage is high", - "name": "TiKV gRPC poll CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of gRPC", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 13 - }, - "hiddenSeries": false, - "id": 105, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"grpc.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 3.6, - "visible": true - } - ], - "timeRegions": [], - "title": "gRPC poll CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 3.6 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0m", - "frequency": "1m", - "handler": 1, - "message": "TiKV scheduler worker thread CPU usage is high", - "name": "TiKV scheduler worker CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of scheduler worker", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 20 - }, - "hiddenSeries": false, - "id": 64, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sched_.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 3.6, - "visible": true - } - ], - "timeRegions": [], - "title": "Scheduler worker CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 3.6 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0m", - "frequency": "1m", - "handler": 1, - "message": "TiKV Storage ReadPool thread CPU usage is high", - "name": "TiKV Storage ReadPool CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of readpool", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 20 - }, - "hiddenSeries": false, - "id": 1908, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_norm.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - normal", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_high.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - high", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "B", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_low.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - low", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "C", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 3.6, - "visible": true - } - ], - "timeRegions": [], - "title": "Storage ReadPool CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 7.2 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0m", - "frequency": "1m", - "handler": 1, - "message": "TiKV unified read pool thread CPU usage is high", - "name": "Unified read pool CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of the unified read pool", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 27 - }, - "hiddenSeries": false, - "id": 4287, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"unified_read_po.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 7.2, - "visible": true - } - ], - "timeRegions": [], - "title": "Unified read pool CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of RocksDB", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 27 - }, - "hiddenSeries": false, - "id": 69, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"rocksdb.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "warning", - "fill": true, - "line": true, - "op": "gt", - "value": 1 - }, - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 4 - } - ], - "timeRegions": [], - "title": "RocksDB CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 7.2 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0m", - "frequency": "1m", - "handler": 1, - "message": "TiKV Coprocessor thread CPU alert", - "name": "TiKV Coprocessor CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of coprocessor", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 34 - }, - "hiddenSeries": false, - "id": 78, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_normal.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} - normal", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_high.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} - high", - "range": true, - "refId": "B", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_low.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} - low", - "range": true, - "refId": "C", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 7.2, - "visible": true - } - ], - "timeRegions": [], - "title": "Coprocessor CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 34 - }, - "hiddenSeries": false, - "id": 2531, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"gc_worker.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "GC worker CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe CPU utilization of split check", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 41 - }, - "hiddenSeries": false, - "id": 68, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"background.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "BackGround Worker CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 41 - }, - "hiddenSeries": false, - "id": 692, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/import-count.*/", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "import-{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance, tid) > 0", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "import-{{instance}}-{{tid}}", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "C", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "import-count-{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "D", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Import CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 48 - }, - "hiddenSeries": false, - "id": 691, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/backup-count.*/", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "backup-{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"backup_endpoint\"}[2m])) by (instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "backup-endpoint", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "B", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[2m])) by (instance, tid) > 0", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "backup-{{instance}}-{{tid}}", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "C", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(tikv_backup_thread_pool_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "backup-count-{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "D", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Backup Worker CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 48 - }, - "hiddenSeries": false, - "id": 62, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*tso/", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cdcwkr.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - worker", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"tso\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - tso", - "range": true, - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CDC worker CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "percentunit", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 55 - }, - "hiddenSeries": false, - "id": 60, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cdc_.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CDC endpoint CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 55 - }, - "hiddenSeries": false, - "id": 23763572511, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftlog_fetch.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raftlog fetch Worker CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of TSO worker", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 62 - }, - "hiddenSeries": false, - "id": 9962, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"tso_worker\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "warning", - "fill": true, - "line": true, - "op": "gt", - "value": 0.3, - "yaxis": "left" - }, - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.8, - "yaxis": "left" - } - ], - "timeRegions": [], - "title": "TSO Worker CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Thread CPU", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 6946, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 7 - }, - "hiddenSeries": false, - "id": 6985, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_ttl_checker_processed_regions{instance=~\"$instance\"}) by (instance) / sum(tikv_raftstore_region_count{instance=~\"$instance\", type=\"region\"}) by (instance)", - "format": "time_series", - "intervalFactor": 10, - "legendFormat": "{{instance}}", - "refId": "E" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TTL check progress", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 7 - }, - "hiddenSeries": false, - "id": 6987, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_ttl_checker_actions{instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "range": true, - "refId": "E" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TTL checker actions", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed when executing GC tasks", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 14 - }, - "hiddenSeries": false, - "id": 6986, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(1, sum(rate(tikv_ttl_checker_compact_duration_bucket{instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "metric": "tikv_storage_command_total", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(tikv_ttl_checker_compact_duration_bucket{instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "tikv_storage_gc_skipped_counter", - "range": true, - "refId": "B", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(tikv_ttl_checker_compact_duration_bucket{instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "range": true, - "refId": "C" - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_ttl_checker_compact_duration_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_ttl_checker_compact_duration_count{instance=~\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "average", - "range": true, - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TTL checker compact duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "rgb(31, 120, 193)", - "mode": "fixed" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 7326, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "expr": "max(tikv_ttl_checker_poll_interval{instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "metric": "tikv_storage_command_total", - "refId": "A", - "step": 4 - } - ], - "title": "TTL checker poll interval", - "type": "stat" - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "TTL", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 2747, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of requests that TiKV sends to PD", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "hiddenSeries": false, - "id": 1069, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ type }}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "PD requests", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed by requests that TiKV sends to PD", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "hiddenSeries": false, - "id": 1070, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_pd_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ type }}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "PD request duration (average)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe total number of PD heartbeat messages", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "hiddenSeries": false, - "id": 1215, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "pending", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ type }}", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "tikv_pd_pending_heartbeat_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "pending", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "PD heartbeats", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total number of peers validated by the PD worker", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "hiddenSeries": false, - "id": 1396, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_pd_validate_peer_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ type }}", - "metric": "", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "PD validate peers", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe count of reconnections between TiKV and PD", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "hiddenSeries": false, - "id": 7985, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(delta(tikv_pd_reconnect_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ type }}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "PD reconnections", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "show": true - }, - { - "format": "opm", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe forward status of PD client", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "hiddenSeries": false, - "id": 8376, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "tikv_pd_request_forwarded{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{host}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "PD forward status", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "opm", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of TSO requests waiting in the queue.", - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 32 - }, - "hiddenSeries": false, - "id": 9963, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "tikv_pd_pending_tso_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Pending TSO Requests", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The slow score of stores", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 32 - }, - "hiddenSeries": false, - "id": 10365, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "tikv_raftstore_slow_score{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store Slow Score", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "opm", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The duration that recorded by inspecting messages.", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 39 - }, - "hiddenSeries": false, - "id": 10366, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_inspect_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance, type))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{type}}", - "range": true, - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Inspected duration per server", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "PD", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 5265, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The throughput of disk write per IO type", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 9 - }, - "hiddenSeries": false, - "id": 5993, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", op=\"write\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", op=\"write\"}[2m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "total", - "range": true, - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Write IO bytes", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": "", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The throughput of disk read per IO type", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 9 - }, - "hiddenSeries": false, - "id": 5994, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", op=\"read\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "sum(rate(tikv_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", op=\"read\"}[2m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "total", - "range": true, - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Read IO bytes", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": "", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The threshold of disk IOs per priority", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 16 - }, - "hiddenSeries": false, - "id": 5995, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_rate_limiter_max_bytes_per_sec{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "IO threshold", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": "", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "IO rate limiter request wait duration.", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 16 - }, - "hiddenSeries": false, - "id": 7225, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(tikv_rate_limiter_request_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": " {{type}}-99%", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "rate(tikv_rate_limiter_request_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tikv_rate_limiter_request_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": " {{type}}-avg", - "range": true, - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Rate Limiter Request Wait Duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "IO Breakdown", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 13117, - "panels": [ - { - "aliasColors": { - "99%": "#eab839", - "999%": "dark-red", - "count": "rgb(33, 250, 2)" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed by processing asynchronous write requests", - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 10 - }, - "hiddenSeries": false, - "id": 13132, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "count", - "dashLength": 1, - "dashes": true, - "fill": 2, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "alias": "avg", - "fill": 7 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "999%", - "range": true, - "refId": "A" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "99%", - "range": true, - "refId": "B" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{instance=~\"$instance\", type=\"write\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\"$instance\", type=\"write\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg", - "range": true, - "refId": "C" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\"$instance\", type=\"write\"}[2m]))", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "count", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Storage async write duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "99%": "#eab839", - "999%": "dark-red", - "count": "rgb(33, 250, 2)" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The store time duration of each request", - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 19 - }, - "hiddenSeries": false, - "id": 13257, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "count", - "dashLength": 1, - "dashes": true, - "fill": 2, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "alias": "avg", - "fill": 7 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "999%", - "range": true, - "refId": "A" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "99%", - "range": true, - "refId": "B" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_duration_secs_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_duration_secs_count{instance=~\"$instance\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg", - "range": true, - "refId": "C" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_duration_secs_count{instance=~\"$instance\"}[2m]))", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "count", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "99%": "#eab839", - "999%": "dark-red", - "count": "rgb(33, 250, 2)" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The apply time duration of each request", - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 19 - }, - "hiddenSeries": false, - "id": 13259, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "count", - "dashLength": 1, - "dashes": true, - "fill": 2, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "alias": "avg", - "fill": 7 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_apply_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "999%", - "range": true, - "refId": "A" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "99%", - "range": true, - "refId": "B" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_apply_duration_secs_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_apply_duration_secs_count{instance=~\"$instance\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg", - "range": true, - "refId": "C" - }, - { - "datasource": null, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_apply_duration_secs_count{instance=~\"$instance\"}[2m]))", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "count", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Apply duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "99%": "#eab839", - "999%": "dark-red", - "count": "rgb(33, 250, 2)" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The propose wait time duration of each request", - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 27 - }, - "hiddenSeries": false, - "id": 13261, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:113", - "alias": "count", - "dashLength": 1, - "dashes": true, - "fill": 2, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "$$hashKey": "object:114", - "alias": "avg", - "fill": 7 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "999%", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "99%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_request_wait_time_duration_secs_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_request_wait_time_duration_secs_count{instance=~\"$instance\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg", - "refId": "C" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_request_wait_time_duration_secs_count{instance=~\"$instance\"}[2m]))", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "count", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store propose wait duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:139", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:140", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "99%": "#eab839", - "999%": "dark-red", - "count": "rgb(33, 250, 2)" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The batch wait time duration of each request", - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 27 - }, - "hiddenSeries": false, - "id": 13263, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:217", - "alias": "count", - "dashLength": 1, - "dashes": true, - "fill": 2, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "$$hashKey": "object:218", - "alias": "avg", - "fill": 7 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "999%", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "99%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg", - "refId": "C" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "count", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store batch wait duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:243", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:244", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "99%": "#eab839", - "999%": "dark-red", - "count": "rgb(33, 250, 2)" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The send-to-write-queue time duration of each request", - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 35 - }, - "hiddenSeries": false, - "id": 13265, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:529", - "alias": "count", - "dashLength": 1, - "dashes": true, - "fill": 2, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "$$hashKey": "object:530", - "alias": "avg", - "fill": 7 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "999%", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "99%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg", - "refId": "C" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "count", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store send to write queue duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:555", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:556", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "99%": "#eab839", - "999%": "dark-red", - "count": "rgb(33, 250, 2)" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The send raft message of the proposal duration of each request", - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 35 - }, - "hiddenSeries": false, - "id": 23763572857, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:529", - "alias": "count", - "dashLength": 1, - "dashes": true, - "fill": 2, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "$$hashKey": "object:530", - "alias": "avg", - "fill": 7 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "999%", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "99%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg", - "refId": "C" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "count", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store send proposal duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:555", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:556", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "99%": "#eab839", - "999%": "dark-red", - "count": "rgb(33, 250, 2)" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The write kv db end duration of each request", - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 43 - }, - "hiddenSeries": false, - "id": 13269, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "count", - "dashLength": 1, - "dashes": true, - "fill": 2, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "alias": "avg", - "fill": 7 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "999%", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "99%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg", - "refId": "C" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "count", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store write kv db end duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "99%": "#eab839", - "999%": "dark-red", - "count": "rgb(33, 250, 2)" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The before write time duration of each request", - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 43 - }, - "hiddenSeries": false, - "id": 13267, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:733", - "alias": "count", - "dashLength": 1, - "dashes": true, - "fill": 2, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "$$hashKey": "object:734", - "alias": "avg", - "fill": 7 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "999%", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "99%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg", - "refId": "C" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "count", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store before write duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:759", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:760", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "99%": "#eab839", - "999%": "dark-red", - "count": "rgb(33, 250, 2)" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The persist duration of each request", - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 51 - }, - "hiddenSeries": false, - "id": 13273, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "count", - "dashLength": 1, - "dashes": true, - "fill": 2, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "alias": "avg", - "fill": 7 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "999%", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "99%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg", - "refId": "C" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "count", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store persist duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "99%": "#eab839", - "999%": "dark-red", - "count": "rgb(33, 250, 2)" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The write end duration of each request", - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 51 - }, - "hiddenSeries": false, - "id": 13271, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:841", - "alias": "count", - "dashLength": 1, - "dashes": true, - "fill": 2, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "$$hashKey": "object:842", - "alias": "avg", - "fill": 7 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "999%", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "99%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg", - "refId": "C" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "count", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store write end duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:867", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:868", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "99%": "#eab839", - "999%": "dark-red", - "count": "rgb(33, 250, 2)" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The commit but not persist duration of each request", - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 59 - }, - "hiddenSeries": false, - "id": 13277, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "count", - "dashLength": 1, - "dashes": true, - "fill": 2, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "alias": "avg", - "fill": 7 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "999%", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "99%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg", - "refId": "C" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "count", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store commit but not persist duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "99%": "#eab839", - "999%": "dark-red", - "count": "rgb(33, 250, 2)" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The commit and persist duration of each request", - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 59 - }, - "hiddenSeries": false, - "id": 13275, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "count", - "dashLength": 1, - "dashes": true, - "fill": 2, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "alias": "avg", - "fill": 7 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "999%", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "99%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg", - "refId": "C" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_count{instance=~\"$instance\"}[2m]))", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "count", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store commit and persist duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Raft Waterfall", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 2748, - "panels": [ - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The time consumed for peer processes to be ready in Raft", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 10 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 13279, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_raftstore_raft_process_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type='ready'}[2m])) by (le)", - "format": "heatmap", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "C", - "step": 4 - } - ], - "title": "Process ready duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed for peer processes to be ready in Raft", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 10 - }, - "hiddenSeries": false, - "id": 13281, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type='ready'}[2m])) by (le, instance))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "C", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 1, - "yaxis": "left" - } - ], - "timeRegions": [], - "title": "99% Process ready duration per server", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dashes": false, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The time duration of store write loop when store-io-pool-size is not zero.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 17 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 13283, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(delta(tikv_raftstore_store_write_loop_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "title": "Store write loop duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time duration of store write loop on each TiKV instance when store-io-pool-size is not zero.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 17 - }, - "hiddenSeries": false, - "id": 13285, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_loop_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} ", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% Store write loop duration per server", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The time consumed when Raft appends log", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 24 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 39, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "title": "Append log duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed when Raft appends log on each TiKV instance", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 24 - }, - "hiddenSeries": false, - "id": 13376, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} ", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_kvdb_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "hide": true, - "interval": "", - "legendFormat": "kvdb-{{instance}}", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_raftdb_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "hide": true, - "interval": "", - "legendFormat": "raftdb-{{instance}}", - "refId": "C" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_send_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "hide": true, - "interval": "", - "legendFormat": "sendmsg-{{instance}}", - "refId": "D" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_callback_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "hide": true, - "interval": "", - "legendFormat": "callback-{{instance}}", - "refId": "E" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% Append log duration per server", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The time consumed when Raft commits log", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 31 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 3690, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Commit log duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed when Raft commits log on each TiKV instance", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 31 - }, - "hiddenSeries": false, - "id": 3688, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% Commit log duration per server", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The time consumed when Raft applies log", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 38 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 31, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "title": "Apply log duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed for Raft to apply logs per TiKV instance", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 38 - }, - "hiddenSeries": false, - "id": 32, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": " {{instance}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% Apply log duration per server", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 45 - }, - "hiddenSeries": false, - "id": 13382, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(tikv_raftstore_io_reschedule_region_total{instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "rechedule-{{instance}}", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(tikv_raftstore_io_reschedule_pending_tasks_total{instance=~\"$instance\"}) by (instance)", - "hide": false, - "interval": "", - "legendFormat": "pending-task-{{instance}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Store io task reschedule", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed when store write task block on each TiKV instance", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 45 - }, - "hiddenSeries": false, - "id": 13380, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_msg_block_wait_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% Write task block duration per server", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Raft IO", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 2751, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The proposal count of all Regions in a mio tick", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 11 - }, - "hiddenSeries": false, - "id": 108, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_proposal_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft proposals per ready", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of proposals per type", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 11 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|normal|read_index\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_raftstore_proposal_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft read/write proposals", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of read proposals which are made by each TiKV instance", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 18 - }, - "hiddenSeries": false, - "id": 119, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|read_index\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft read proposals per server", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of write proposals which are made by each TiKV instance", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 18 - }, - "hiddenSeries": false, - "id": 120, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"normal\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_raftstore_proposal_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft write proposals per server", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The wait time of each proposal", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 25 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 41, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "metric": "tikv_raftstore_request_wait_time_duration_secs_bucket", - "refId": "A", - "step": 4 - } - ], - "title": "Propose wait duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The wait time of each proposal in each TiKV instance", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 25 - }, - "hiddenSeries": false, - "id": 42, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% Propose wait duration per server", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dashes": false, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The wait time of each store write task", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 32 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 13524, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(delta(tikv_raftstore_store_write_task_wait_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "metric": "tikv_raftstore_request_wait_time_duration_secs_bucket", - "refId": "A", - "step": 4 - } - ], - "title": "Store write wait duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The wait time of each store write task in each TiKV instance", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 32 - }, - "hiddenSeries": false, - "id": 13522, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_task_wait_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% Store write wait duration per server", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 39 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 2535, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "metric": "tikv_raftstore_request_wait_time_duration_secs_bucket", - "refId": "A", - "step": 4 - } - ], - "title": "Apply wait duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 39 - }, - "hiddenSeries": false, - "id": 2536, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% Apply wait duration per server", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dashes": false, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The handle duration of each store write task msg", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 46 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 23763572700, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(delta(tikv_raftstore_store_write_handle_msg_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "metric": "tikv_raftstore_request_wait_time_duration_secs_bucket", - "refId": "A", - "step": 4 - } - ], - "title": "Store write handle msg duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dashes": false, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The distribution of write trigger size", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 46 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 23763572701, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(delta(tikv_raftstore_store_write_trigger_wb_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "metric": "tikv_raftstore_request_wait_time_duration_secs_bucket", - "refId": "A", - "step": 4 - } - ], - "title": "Store write trigger size", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "format": "bytes", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The rate at which peers propose logs", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 54 - }, - "hiddenSeries": false, - "id": 1975, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(rate(tikv_raftstore_propose_log_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft propose speed", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "bytes/s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 54 - }, - "hiddenSeries": false, - "id": 1976, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "store-{{type}}", - "metric": "tikv_raftstore_request_wait_time_duration_secs_bucket", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "apply-{{type}}", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Perf Context duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Raft Propose", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 2749, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of different ready type of Raft", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 - }, - "hiddenSeries": false, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_raft_ready_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_raftstore_raft_ready_handled_total", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_raft_process_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"ready\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "count", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Ready handled", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The max time consumed by raftstore events", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 - }, - "hiddenSeries": false, - "id": 123, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(1.0, sum(rate(tikv_raftstore_event_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "C", - "step": 4 - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(1.0, sum(rate(tikv_broadcast_normal_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "broadcast_normal", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 1 - } - ], - "timeRegions": [], - "title": "Max duration of raft store events", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The time consumed for checking memory locks for replica reads", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 21 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 7235, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_replica_read_lock_check_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "C", - "step": 4 - } - ], - "title": "Replica read lock checking duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The length of peer msgs for each round handling", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 21 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 23763572958, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(delta(tikv_raftstore_peer_msg_len_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "C", - "step": 4 - } - ], - "title": "Peer msg length distribution", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "none", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Raft Process", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 2750, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of Raft messages sent by each TiKV instance", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 29 - }, - "hiddenSeries": false, - "id": 1615, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Sent messages per server", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of Raft messages flushed by each TiKV instance", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 29 - }, - "hiddenSeries": false, - "id": 1616, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_server_raft_message_flush_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{reason}}", - "metric": "tikv_server_raft_message_flush_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Flush messages per server", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of Raft messages received by each TiKV instance", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 36 - }, - "hiddenSeries": false, - "id": 106, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_server_raft_message_recv_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Receive messages per server", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of different types of Raft messages that are sent", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 36 - }, - "hiddenSeries": false, - "id": 11, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Messages", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total number of vote messages that are sent in Raft", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 43 - }, - "hiddenSeries": false, - "id": 25, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"vote\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Vote", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of dropped Raft messages per type", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 43 - }, - "hiddenSeries": false, - "id": 1309, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_raft_dropped_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft dropped messages", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Raft Message", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 2752, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of admin proposals", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 76, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"conf_change|transfer_leader\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_raftstore_proposal_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Admin proposals", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of the processed apply command", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 77, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_admin_cmd_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", status=\"success\", type!=\"compact\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_raftstore_admin_cmd_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Admin apply", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of raftstore split checksss", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 70, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_check_split_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"ignore\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_raftstore_check_split_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Check split", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed when running split check in .9999", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 71, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.9999, sum(rate(tikv_raftstore_check_split_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_raftstore_check_split_duration_seconds_bucket", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99.99% Check split duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 25 - }, - "hiddenSeries": false, - "id": 3636, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 400, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_load_base_split_event[2m])) by (type)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_raftstore_check_split_total", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "", - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Load base split event", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 118 - }, - "hiddenSeries": false, - "id": 23763572060, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.80, sum(rate(tikv_load_base_split_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "80%-{{instance}}", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.90, sum(rate(tikv_load_base_split_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}[2m])) by (le, instance))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "99%-{{instance}}", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_load_base_split_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}[2m])) by (instance)", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "avg-{{instance}}", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Load base split duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:270", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:271", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 25 - }, - "hiddenSeries": false, - "id": 3637, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "tikv_read_qps_topn{order=\"0\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_raftstore_check_split_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TopN QPS exceeds threshold", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Raft Admin", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 12797, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "hiddenSeries": false, - "id": 12882, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "99% - {{instance}}", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "95% - {{instance}}", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance) / sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "avg - {{instance}}", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft log GC write duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 10, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "hiddenSeries": false, - "id": 12886, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "99% - {{instance}}", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "95% - {{instance}}", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance) / sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "avg - {{instance}}", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft log GC kv sync duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 10, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "hiddenSeries": false, - "id": 12881, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_raftstore_check_split_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft log GC write operations ", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "hiddenSeries": false, - "id": 12884, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_raft_log_gc_seek_operations_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_raftstore_check_split_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft log GC seek operations ", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "hiddenSeries": false, - "id": 12887, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_log_lag_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_raftstore_check_split_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft log lag", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "hiddenSeries": false, - "id": 12975, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_raft_log_gc_skipped{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{reason}}", - "metric": "tikv_raftstore_check_split_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft log gc skipped", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "hiddenSeries": false, - "id": 12974, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_raft_log_gc_failed{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_raftstore_check_split_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft log GC failed", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "hiddenSeries": false, - "id": 23763572229, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_entry_fetches{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_raftstore_check_split_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft log fetch", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "hiddenSeries": false, - "id": 23763572555, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": false - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:521", - "alias": "/pending-task/", - "transform": "negative-Y", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(tikv_worker_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\", name=~\"raftlog-fetch-worker\"})", - "hide": false, - "interval": "", - "legendFormat": "pending-task", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Raft log async fetch task duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:86", - "format": "s", - "logBase": 10, - "show": true - }, - { - "$$hashKey": "object:87", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Raft Log", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 2753, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The number of rejections from the local read thread and The number of total requests", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 12 - }, - "id": 2292, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*-total/i", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_local_read_reject_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-reject-by-{{reason}}", - "refId": "A" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_local_read_executed_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-total", - "refId": "B" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_local_read_executed_stale_read_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-stale-read", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Local reader requests", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Local reader", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 4200, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time used by each level in the unified read pool per second. Level 0 refers to small queries.", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 13 - }, - "id": 4194, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_multilevel_level_elapsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (level)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{level}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Time used by level", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The chance that level 0 (small) tasks are scheduled in the unified read pool.", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 13 - }, - "id": 4196, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "tikv_multilevel_level0_chance{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Level 0 chance", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The number of concurrently running tasks in the unified read pool.", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 21 - }, - "id": 4198, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(avg_over_time(tikv_unified_read_pool_running_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Running tasks", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 26 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 23763572469, - "legend": { - "show": false - }, - "pluginVersion": "7.5.11", - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_yatp_pool_schedule_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"unified-read.*\"}[2m])) by (le)", - "format": "heatmap", - "interval": "", - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "Unified Read Pool Wait Duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Unified read pool task execution time during one schedule.", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 34 - }, - "id": 4199, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.50, sum(rate(tikv_yatp_task_poll_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "50%", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(tikv_yatp_task_poll_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_yatp_task_poll_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "C" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_yatp_task_poll_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "999%", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Duration of One Time Slice", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Unified read pool task total execution duration.", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 34 - }, - "id": 4202, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.50, sum(rate(tikv_yatp_task_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "50%", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(tikv_yatp_task_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_yatp_task_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "C" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_yatp_task_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "999%", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Task Execute Duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Task schedule number of times.", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 42 - }, - "id": 4204, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.50, sum(rate(tikv_yatp_task_execute_times_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "50%", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(tikv_yatp_task_execute_times_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_yatp_task_execute_times_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "C" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_yatp_task_execute_times_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "999%", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Task Schedule Times", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Unified Read Pool", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 2754, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total count of different kinds of commands received", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 14 - }, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Storage command total", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 10, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total number of engine asynchronous request errors", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 8, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", status!~\"all|success\"}[2m])) by (status)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{status}}", - "metric": "tikv_raftstore_raft_process_duration_secs_bucket", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Storage async request error", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The time consumed by processing asynchronous snapshot requests", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 22 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 15, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le)", - "format": "heatmap", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A", - "step": 4 - } - ], - "title": "Storage async snapshot duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The time consumed by processing asynchronous write requests", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 22 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 109, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le)", - "format": "heatmap", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A", - "step": 4 - } - ], - "title": "Storage async write duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The storage async snapshot duration", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 35 - }, - "hiddenSeries": false, - "id": 20000, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.10", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", - "interval": "", - "intervalFactor": 2, - "legendFormat": "99%", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m]))", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Storage async snapshot duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:295", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:296", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The storage async write duration", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 35 - }, - "hiddenSeries": false, - "id": 20001, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.10", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le))", - "interval": "", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le))", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m]))", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "avg", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Storage async write duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:494", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:495", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Storage", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 9160, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 20 - }, - "height": "", - "hiddenSeries": false, - "id": 9552, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "maxPerRow": 2, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_scheduler_write_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "write-{{instance}}", - "metric": "", - "refId": "A", - "step": 40 - }, - { - "datasource": null, - "expr": "sum(tikv_scheduler_throttle_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "throttle-{{instance}}", - "metric": "", - "refId": "B", - "step": 40 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler flow", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 20 - }, - "height": "", - "hiddenSeries": false, - "id": 9553, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "maxPerRow": 2, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_scheduler_discard_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) / 10000000", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "", - "refId": "A", - "step": 40 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler discard ratio", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 28 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 11512, - "legend": { - "show": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_scheduler_throttle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Throttle duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of pending commands per TiKV instance", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 28 - }, - "height": "", - "hiddenSeries": false, - "id": 11906, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 2, - "links": [], - "maxPerRow": 2, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "tikv_scheduler_throttle_cf{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} != 0", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{cf}}", - "metric": "", - "refId": "A", - "step": 40 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler throttled CF", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 36 - }, - "hiddenSeries": false, - "id": 9947, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_scheduler_throttle_action_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (cf, type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}-{{cf}}", - "metric": "tikv_grpc_msg_duration_seconds_bucket", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Flow controller actions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 36 - }, - "height": "", - "hiddenSeries": false, - "id": 10338, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "maxPerRow": 2, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_scheduler_l0_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, cf)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cf}}_l0_flow-{{instance}}", - "metric": "", - "refId": "D", - "step": 40 - }, - { - "datasource": null, - "expr": "sum(tikv_scheduler_flush_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, cf)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cf}}_flush_flow-{{instance}}", - "metric": "", - "refId": "E", - "step": 40 - }, - { - "datasource": null, - "expr": "sum(tikv_scheduler_l0_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "total_l0_flow-{{instance}}", - "metric": "", - "refId": "B", - "step": 40 - }, - { - "datasource": null, - "expr": "sum(tikv_scheduler_flush_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "total_flush_flow-{{instance}}", - "metric": "", - "refId": "C", - "step": 40 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Flush/L0 flow", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 44 - }, - "height": "", - "hiddenSeries": false, - "id": 9944, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "maxPerRow": 2, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "max(tikv_scheduler_l0{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "l0-{{instance}}", - "metric": "", - "refId": "A", - "step": 40 - }, - { - "datasource": null, - "expr": "max(tikv_scheduler_memtable{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memtable-{{instance}}", - "metric": "", - "refId": "B", - "step": 40 - }, - { - "datasource": null, - "expr": "max(tikv_scheduler_l0_avg{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg_l0-{{instance}}", - "metric": "", - "refId": "C", - "step": 40 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Flow controller factors", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 44 - }, - "hiddenSeries": false, - "id": 9946, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "cf", - "yaxis": 2 - }, - { - "alias": "pending-bytes", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_engine_pending_compaction_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\"}) by (cf)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{cf}}", - "metric": "tikv_engine_pending_compaction_bytes", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "exemplar": true, - "expr": "max(tikv_scheduler_pending_compaction_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) / 10000000", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "pending-bytes-{{instance}}", - "metric": "tikv_engine_pending_compaction_bytes", - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Compaction pending bytes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "30", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Throttle time for txn storage commands in 1 minute.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 52 - }, - "hiddenSeries": false, - "id": 23763572363, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_txn_command_throttle_time_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Txn command throttled duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Throttle time for non-txn related processing like analyze or dag in 1 minute.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 52 - }, - "hiddenSeries": false, - "id": 23763572365, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_non_txn_command_throttle_time_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) ", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Non-txn command throttled duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Flow Control", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 2756, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total number of commands on each stage in commit command", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 18 - }, - "height": "400", - "hiddenSeries": false, - "id": 168, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxPerRow": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "busy", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (stage)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{stage}}", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler stage total", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed when executing commit command", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 28 - }, - "hiddenSeries": false, - "id": 3, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_scheduler_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tikv_scheduler_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler command duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time which is caused by latch wait in commit command", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 28 - }, - "hiddenSeries": false, - "id": 194, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_scheduler_latch_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tikv_scheduler_latch_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler latch wait duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of keys read by a commit command", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 36 - }, - "hiddenSeries": false, - "id": 195, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "kv_command_key", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_scheduler_kv_command_key_read_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tikv_scheduler_kv_command_key_read_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler keys read", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of keys written by a commit command", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 36 - }, - "hiddenSeries": false, - "id": 373, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "kv_command_key", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_scheduler_kv_command_key_write_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tikv_scheduler_kv_command_key_write_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler keys written", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The keys scan details of each CF when executing commit command", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 44 - }, - "id": 560, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\"}[2m])) by (tag)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{tag}}", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler scan details", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The keys scan details of lock CF when executing commit command", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 44 - }, - "id": 675, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"lock\"}[2m])) by (tag)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{tag}}", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler scan details [lock]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The keys scan details of write CF when executing commit command", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 52 - }, - "id": 829, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"write\"}[2m])) by (tag)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{tag}}", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler scan details [write]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The keys scan details of default CF when executing commit command", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 52 - }, - "id": 830, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"default\"}[2m])) by (tag)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{tag}}", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler scan details [default]", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed on reading when executing commit command", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 63 - }, - "hiddenSeries": false, - "id": 23763572710, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_processing_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_processing_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "95%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_scheduler_processing_read_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tikv_scheduler_processing_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) ", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler command read duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:95", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:96", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The time consumed on checking memory locks", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 63 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 7236, - "legend": { - "show": false - }, - "links": [], - "repeatedByRow": true, - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_storage_check_mem_lock_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le)", - "format": "heatmap", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "title": "Check memory locks duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - } - ], - "repeat": "command", - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Scheduler - $command", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 2755, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total number of commands on each stage", - "fill": 1, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 15 - }, - "height": "400", - "id": 167, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxPerRow": 1, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (stage)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "busy", - "refId": "A", - "step": 20 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (stage)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{stage}}", - "refId": "B", - "step": 20 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler stage total", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total writing bytes of commands on each stage", - "fill": 1, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 15 - }, - "height": "400", - "id": 3834, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxPerRow": 1, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_scheduler_writing_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 20 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler writing bytes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "bytes", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of different priority commands", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 25 - }, - "height": "", - "id": 1, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "maxPerRow": 2, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_scheduler_commands_pri_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (priority)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{priority}}", - "metric": "", - "refId": "A", - "step": 40 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scheduler priority commands", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 300 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "frequency": "120s", - "handler": 1, - "message": "TiKV scheduler context total", - "name": "scheduler pending commands alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of pending commands per TiKV instance", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 25 - }, - "height": "", - "id": 193, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "maxPerRow": 2, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_scheduler_contex_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "", - "refId": "A", - "step": 40 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 300 - } - ], - "timeRegions": [], - "title": "Scheduler pending commands", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 45 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 23763572468, - "legend": { - "show": false - }, - "pluginVersion": "7.5.11", - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_yatp_pool_schedule_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sched-worker.*\"}[2m])) by (le)", - "format": "heatmap", - "interval": "", - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "Txn Scheduler Pool Wait Duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Scheduler", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 2758, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of GC tasks processed by gc_worker", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 46 - }, - "id": 121, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_gcworker_gc_tasks_vec{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (task)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "total-{{task}}", - "metric": "tikv_storage_command_total", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_storage_gc_skipped_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (task)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "skipped-{{task}}", - "metric": "tikv_storage_gc_skipped_counter", - "refId": "B", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_gcworker_gc_task_fail_vec{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (task)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "failed-{{task}}", - "refId": "C" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_gc_worker_too_busy{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "gcworker-too-busy", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "GC tasks", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed when executing GC tasks", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 46 - }, - "id": 2224, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(1, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, task))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max-{{task}}", - "metric": "tikv_storage_command_total", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, task))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%-{{task}}", - "metric": "tikv_storage_gc_skipped_counter", - "refId": "B", - "step": 4 - }, - { - "datasource": null, - "expr": "histogram_quantile(0.95, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, task))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%-{{task}}", - "refId": "C" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_gcworker_gc_task_duration_vec_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (task) / sum(rate(tikv_gcworker_gc_task_duration_vec_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (task)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "average-{{task}}", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "GC tasks duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The GC duration", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 969, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(1.0, sum(rate(tidb_tikvclient_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (instance, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 40 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TiDB GC seconds", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "The count of TiDB GC worker actions", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 53 - }, - "id": 966, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tidb_tikvclient_gc_worker_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TiDB GC worker actions", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "description": "Progress of ResolveLocks, the first phase of GC", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 60 - }, - "id": 2823, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "max(tidb_tikvclient_range_task_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=~\"resolve-locks.*\"}) by (result)", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{result}}", - "metric": "tikv_storage_command_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "ResolveLocks Progress", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "description": "Progress of TiKV's GC", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 60 - }, - "id": 2821, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_gcworker_autogc_processed_regions{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan\"}) by (instance) / sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_storage_command_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TiKV Auto GC Progress", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "max": "1.1", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "keys / second", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 2589, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_storage_mvcc_gc_delete_versions_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{key_mode}}_keys/s", - "refId": "E" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "GC speed", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "description": "SafePoint used for TiKV's Auto GC", - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 2822, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "max(tikv_gcworker_autogc_safe_point{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance) / (2^18)", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_storage_command_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TiKV Auto GC SafePoint", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "dateTimeAsIso", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "datasource": null, - "description": " \tThe lifetime of TiDB GC", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 74 - }, - "id": 27, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "expr": "max(tidb_tikvclient_gc_config{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tikv_gc_life_time\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "refId": "A", - "step": 60 - } - ], - "title": "GC lifetime", - "type": "stat" - }, - { - "datasource": null, - "description": "The interval of TiDB GC", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 74 - }, - "id": 28, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.1.1", - "targets": [ - { - "datasource": null, - "expr": "max(tidb_tikvclient_gc_config{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tikv_gc_run_interval\"})", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 60 - } - ], - "title": "GC interval", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "description": "Keys handled in GC compaction filter", - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 74 - }, - "id": 6596, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_gc_compaction_filtered{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{key_mode}}_filtered", - "metric": "tikv_storage_command_total", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_gc_compaction_filter_skip{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{key_mode}}_skipped", - "refId": "B" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_gc_compaction_mvcc_rollback{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{key_mode}}_mvcc-rollback/mvcc-lock", - "refId": "C" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_gc_compaction_filter_orphan_versions{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{key_mode}}_orphan-versions", - "refId": "D" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_gc_compaction_filter_perform{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{key_mode}}_performed-times", - "refId": "E" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_gc_compaction_failure{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode,type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{key_mode}}_failure-{{type}}", - "refId": "F" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_gc_compaction_filter_mvcc_deletion_met{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{key_mode}}_mvcc-deletion-met", - "refId": "G" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_gc_compaction_filter_mvcc_deletion_handled{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{key_mode}}_mvcc-deletion-handled", - "refId": "H" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_gc_compaction_filter_mvcc_deletion_wasted{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{key_mode}}_mvcc-deletion-wasted", - "refId": "I" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "GC in Compaction Filter", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "description": "GC scan write details", - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 86 - }, - "id": 8767, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_gcworker_gc_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\", cf=\"write\"}[2m])) by (key_mode,tag)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{key_mode}}_{{tag}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "GC scan write details", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "description": "GC scan default details", - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 86 - }, - "id": 8768, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_gcworker_gc_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\", cf=\"default\"}[2m])) by (key_mode,tag)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{key_mode}}_{{tag}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "GC scan default details", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "GC", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 29 - }, - "id": 2759, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The rate of Raft snapshot messages sent", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 20 - }, - "id": 35, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": " ", - "refId": "A", - "step": 60 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Rate snapshot message", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed when handling snapshots", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 20 - }, - "id": 37, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_server_send_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "send", - "refId": "A", - "step": 60 - }, - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"apply\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "apply", - "refId": "B", - "step": 60 - }, - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"generate\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "generate", - "refId": "C", - "step": 60 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% Handle snapshot duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The number of snapshots in different states", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 20 - }, - "id": 38, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_raftstore_snapshot_traffic_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "", - "refId": "A", - "step": 60 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Snapshot state count", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The snapshot size (P99.99).9999", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 27 - }, - "id": 44, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "size", - "metric": "tikv_snapshot_size_bucket", - "refId": "A", - "step": 40 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99.99% Snapshot size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The number of KV within a snapshot in .9999", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 27 - }, - "id": 43, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_kv_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "count", - "metric": "tikv_snapshot_kv_count_bucket", - "refId": "A", - "step": 40 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99.99% Snapshot KV count", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Action stats for snapshot generating and applying", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 66 - }, - "id": 36, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_raftstore_snapshot_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, status)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{type}}-{{status}}", - "refId": "A" - }, - { - "datasource": null, - "expr": "sum(delta(tikv_raftstore_clean_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "clean-region-by-{{type}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Snapshot Actions", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Snapshot", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 2760, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of tasks handled by worker", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 21 - }, - "id": 59, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_worker_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Worker handled tasks", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tCurrent pending and running tasks of worker", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 21 - }, - "id": 1395, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_worker_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Worker pending tasks", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of tasks handled by future_pool", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 29 - }, - "id": 1876, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_futurepool_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "FuturePool handled tasks", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "Current pending and running tasks of future_pool", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 29 - }, - "id": 1877, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_futurepool_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "FuturePool pending tasks", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Task", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 2757, - "panels": [ - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#5195ce", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The time consumed to handle coprocessor read requests", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 22 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 3062, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": false, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Request duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 22 - }, - "id": 16, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-100%", - "refId": "E" - }, - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-99%", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Request duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 1, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 30 - }, - "id": 74, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_coprocessor_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (req)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{req}}", - "metric": "tikv_coprocessor_request_error", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total Requests", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "ops", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 30 - }, - "id": 3128, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (reason)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{reason}}", - "metric": "tikv_coprocessor_request_error", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total Request Errors", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "ops", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 37 - }, - "id": 52, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_coprocessor_scan_keys_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (req)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total KV Cursor Operations", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 37 - }, - "id": 3129, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(1, avg(rate(tikv_coprocessor_scan_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, req)) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "100%-{{req}}", - "refId": "D" - }, - { - "datasource": null, - "expr": "histogram_quantile(0.99, avg(rate(tikv_coprocessor_scan_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, req)) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%-{{req}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "KV Cursor Operations", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 44 - }, - "id": 2118, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "key_skipped", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_coprocessor_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\" ,metric=\"internal_delete_skipped_count\"}[2m])) by (req)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "delete_skipped-{{req}}", - "metric": "scan_details", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total RocksDB Perf Statistics", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 44 - }, - "id": 551, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_coprocessor_response_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "size", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total Response Size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "decbytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Coprocessor Overview", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 3197, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed when handling coprocessor requests", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 113, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-100%", - "refId": "E" - }, - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-99%", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Handle duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 1, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "s", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "decimals": 1, - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed to handle coprocessor requests per TiKV instance (P95)", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 23 - }, - "id": 117, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{req}}", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "95% Handle duration by store", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed when coprocessor requests are wait for being handled", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 30 - }, - "id": 111, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-100%", - "refId": "D" - }, - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-99%", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Wait duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed when coprocessor requests are wait for being handled in each TiKV instance", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 30 - }, - "id": 116, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{req}}", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "95% Wait duration by store", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "decimals": 1, - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 37 - }, - "id": 3195, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_coprocessor_dag_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (vec_type)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{vec_type}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total DAG Requests", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total number of DAG executors", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 37 - }, - "id": 3264, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_coprocessor_executor_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total DAG Executors", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 44 - }, - "id": 552, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_coprocessor_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"select\"}[2m])) by (tag)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{tag}}", - "metric": "scan_details", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total Ops Details (Table Scan)", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 44 - }, - "id": 3263, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_coprocessor_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"index\"}[2m])) by (tag)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{tag}}", - "metric": "scan_details", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total Ops Details (Index Scan)", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 122, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_coprocessor_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"select\"}[2m])) by (tag,cf)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{cf}}-{{tag}}", - "metric": "scan_details", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total Ops Details by CF (Table Scan)", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 53 - }, - "id": 554, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": "cf", - "repeatDirection": "h", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_coprocessor_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"index\"}[2m])) by (tag,cf)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{cf}}-{{tag}}", - "metric": "scan_details", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total Ops Details by CF (Index Scan)", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#5195ce", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The time consumed on checking memory locks for coprocessor requests", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 119 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 7594, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_coprocessor_mem_lock_check_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Memory lock checking duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Coprocessor Detail", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 2761, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 2108, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, state)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{state}}", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(tikv_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-total", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Threads state", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 2258, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "topk(20, sum(rate(tikv_threads_io_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name, io) > 1024)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{name}}-{{io}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Threads IO", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 31 - }, - "id": 2660, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "topk(20, max(rate(tikv_thread_voluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name) > 200)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{name}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Thread Voluntary Context Switches", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 31 - }, - "id": 2661, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "topk(20, max(rate(tikv_thread_nonvoluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name) > 100)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{name}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Thread Nonvoluntary Context Switches", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Threads", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 34 - }, - "id": 2762, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of get operations", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 59 - }, - "hiddenSeries": false, - "id": 138, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memtable", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "block_cache", - "metric": "", - "refId": "E", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l0\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "l0", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l1\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "l1", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l2_and_up\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "l2_and_up", - "refId": "F", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Get operations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed when executing get operations", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 59 - }, - "hiddenSeries": false, - "id": 82, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "max(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Get duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of seek operations", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 67 - }, - "hiddenSeries": false, - "id": 129, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "seek", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek_found\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "seek_found", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "next", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next_found\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "next_found", - "metric": "", - "refId": "D", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "prev", - "metric": "", - "refId": "E", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev_found\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "prev_found", - "metric": "", - "refId": "F", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Seek operations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed when executing seek operation", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 67 - }, - "hiddenSeries": false, - "id": 125, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "max(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Seek duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of write operations", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 75 - }, - "hiddenSeries": false, - "id": 139, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "done", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "timeout", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "with_wal", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Write operations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed when executing write operation", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 75 - }, - "hiddenSeries": false, - "id": 126, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "max(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Write duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe count of WAL sync operations", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 83 - }, - "hiddenSeries": false, - "id": 137, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_wal_file_synced{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "sync", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "WAL sync operations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed when executing write wal operation", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 83 - }, - "hiddenSeries": false, - "id": 130, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "max(tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Write WAL duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of compaction and flush operations", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 91 - }, - "hiddenSeries": false, - "id": 128, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_engine_event_total", - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Compaction operations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed when executing WAL sync operation", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 91 - }, - "hiddenSeries": false, - "id": 135, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxPerRow": 2, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "max(tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "WAL sync duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 10, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "Compaction guard actions", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 99 - }, - "hiddenSeries": false, - "id": 2453, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_raftstore_compaction_guard_action_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", cf=~\"default|write\"}[2m])) by (cf, type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cf}}-{{type}}", - "metric": "", - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Compaction guard actions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed when executing the compaction and flush operations", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 99 - }, - "hiddenSeries": false, - "id": 136, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "max(tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Compaction duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed when reading SST files", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 107 - }, - "hiddenSeries": false, - "id": 140, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "max(tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "SST read duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 10, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 107 - }, - "hiddenSeries": false, - "id": 2451, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_compaction_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (cf, reason)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{cf}} - {{reason}}", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Compaction reason", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The block cache size. Broken down by column family if shared block cache is disabled.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 115 - }, - "hiddenSeries": false, - "id": 102, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "topk(20, avg(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by(cf, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{cf}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Block cache size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The hit rate of memtable", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 115 - }, - "hiddenSeries": false, - "id": 88, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[2m])) / (sum(rate(tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"$db\", type=\"memtable_hit\"}[2m])) + sum(rate(tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"$db\", type=\"memtable_miss\"}[2m])))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "hit", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memtable hit", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "ops", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The flow of different kinds of block cache operations", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 123 - }, - "height": "", - "hiddenSeries": false, - "id": 467, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "total_read", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "total_written", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "data_insert", - "metric": "", - "refId": "D", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "filter_insert", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "filter_evict", - "metric": "", - "refId": "E", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "index_insert", - "metric": "", - "refId": "F", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "index_evict", - "metric": "", - "refId": "G", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Block cache flow", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 10, - "min": "0", - "show": true - }, - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The hit rate of block cache", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 123 - }, - "hiddenSeries": false, - "id": 80, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxPerRow": 2, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_miss\"}[2m])))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "all", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_miss\"}[2m])))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "data", - "metric": "", - "refId": "D", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_miss\"}[2m])))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "filter", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_miss\"}[2m])))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "index", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_useful\"}[2m])) / sum(rate(tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_checked\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "bloom prefix", - "metric": "", - "refId": "E", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Block cache hit", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "ops", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The flow of different kinds of operations on keys", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 131 - }, - "height": "", - "hiddenSeries": false, - "id": 132, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "read", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "written", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_compaction_num_corrupt_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "corrupt", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Keys flow", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of different kinds of block cache operations", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 131 - }, - "hiddenSeries": false, - "id": 468, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "total_add", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_add\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "data_add", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_add\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "filter_add", - "metric": "", - "refId": "D", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_add\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "index_add", - "metric": "", - "refId": "E", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add_failures\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "add_failures", - "metric": "", - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Block cache operations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The flow rate of read operations per type", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 139 - }, - "height": "", - "hiddenSeries": false, - "id": 85, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "get", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "scan", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Read flow", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of keys in each column family", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 139 - }, - "hiddenSeries": false, - "id": 131, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_engine_estimate_num_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{cf}}", - "metric": "tikv_engine_estimate_num_keys", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total keys", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The flow of different kinds of write operations", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 147 - }, - "height": "", - "hiddenSeries": false, - "id": 86, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[2m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "wal", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "write", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Write flow", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The bytes per read", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 147 - }, - "hiddenSeries": false, - "id": 133, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxPerRow": 2, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "max(tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Bytes / Read", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "logBase": 10, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The flow rate of compaction operations per type", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 155 - }, - "hiddenSeries": false, - "id": 90, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "read", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "written", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[2m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "flushed", - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Compaction flow", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "Bps", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The bytes per write", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 155 - }, - "hiddenSeries": false, - "id": 134, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxPerRow": 2, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "max(tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Bytes / Write", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "logBase": 10, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The read amplification per TiKV instance \t", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 163 - }, - "hiddenSeries": false, - "id": 518, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[2m])) by (instance) / sum(rate(tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Read amplication", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The pending bytes to be compacted", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 163 - }, - "hiddenSeries": false, - "id": 127, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_engine_pending_compaction_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{cf}}", - "metric": "tikv_engine_pending_compaction_bytes", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Compaction pending bytes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "Bps", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of snapshot of each TiKV instance", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 171 - }, - "hiddenSeries": false, - "id": 516, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "tikv_engine_num_snapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Number of snapshots", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The compression ratio of each level", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 171 - }, - "hiddenSeries": false, - "id": 863, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf, level)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{cf}} - level - {{level}}", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Compression ratio", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The number of SST files for different column families in each level", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 179 - }, - "hiddenSeries": false, - "id": 2002, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_num_files_at_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf, level)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "cf-{{cf}}, level-{{level}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Number files at each level", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time that the oldest unreleased snapshot survivals", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 179 - }, - "hiddenSeries": false, - "id": 517, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "tikv_engine_oldest_snapshot_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_engine_oldest_snapshot_duration", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Oldest snapshots duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Stall conditions changed of each column family", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 187 - }, - "hiddenSeries": false, - "id": 2381, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "tikv_engine_stall_conditions_changed{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{cf}}-{{type}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Stall conditions changed of each CF", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time consumed when ingesting SST files", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 187 - }, - "hiddenSeries": false, - "id": 2003, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_snapshot_ingest_sst_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "A" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_snapshot_ingest_sst_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) / sum(rate(tikv_snapshot_ingest_sst_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "average", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Ingest SST duration seconds", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 195 - }, - "hiddenSeries": false, - "id": 2452, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(increase(tikv_engine_write_stall_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (type)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Write Stall Reason", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time which is caused by write stall", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 195 - }, - "hiddenSeries": false, - "id": 87, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "max(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "datasource": null, - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Write stall duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 10, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The level that the external file ingests into", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 203 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 12712, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(delta(tikv_engine_ingestion_picked_level_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (le)", - "format": "heatmap", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "title": "Ingestion picked level", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The memtable size of each column family", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 203 - }, - "hiddenSeries": false, - "id": 103, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"mem-tables\"}) by (cf)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cf}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memtable size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "repeat": "db", - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "RocksDB - $db", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "id": 12802, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The count of operations per second", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "hiddenSeries": false, - "id": 12892, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(raft_engine_write_apply_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "interval": "", - "legendFormat": "write", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(raft_engine_read_entry_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "read_entry", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(raft_engine_read_message_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "read_message", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Operation", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The time used in write operation", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "hiddenSeries": false, - "id": 12893, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(raft_engine_write_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(raft_engine_write_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "interval": "", - "legendFormat": "avg", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(raft_engine_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "95%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(raft_engine_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "99%", - "refId": "C" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "999%", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Write Duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The I/O flow rate", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "hiddenSeries": false, - "id": 12896, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(raft_engine_write_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "interval": "", - "legendFormat": "write", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(raft_engine_background_rewrite_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "hide": false, - "interval": "", - "legendFormat": "rewrite {{type}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Flow", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "binBps", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "99% duration breakdown of write operation", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "hiddenSeries": false, - "id": 12895, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(raft_engine_write_preprocess_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "wait", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(raft_engine_write_leader_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "wal", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(raft_engine_write_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "apply", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Write Duration Breakdown (99%)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The bytes per write", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "hiddenSeries": false, - "id": 12898, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(raft_engine_write_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(raft_engine_write_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "interval": "", - "legendFormat": "avg", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(raft_engine_write_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "99%", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(raft_engine_write_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "999%", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Bytes / Written", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "999% duration breakdown of WAL write operation", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "hiddenSeries": false, - "id": 12933, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(raft_engine_write_leader_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "total", - "refId": "D" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(raft_engine_sync_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "sync", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(raft_engine_allocate_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "allocate", - "refId": "G" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(raft_engine_rotate_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "rotate", - "refId": "H" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "WAL Duration Breakdown (999%)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The average number of files", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 56 - }, - "hiddenSeries": false, - "id": 12899, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "avg(raft_engine_log_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "avg(raft_engine_swap_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", - "intervalFactor": 1, - "legendFormat": "swap", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "File Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The 99% duration of operations other than write", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "hiddenSeries": false, - "id": 12897, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(raft_engine_read_entry_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "read_entry", - "refId": "B" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(raft_engine_read_message_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "read_message", - "refId": "D" - }, - { - "datasource": null, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(raft_engine_purge_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "purge", - "refId": "E" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Other Durations (99%)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The average number of log entries", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 64 - }, - "hiddenSeries": false, - "id": 12934, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "avg(raft_engine_log_entry_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", - "hide": false, - "interval": "", - "legendFormat": "{{type}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Entry Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Raft Engine", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 3301, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 27 - }, - "id": 3555, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_engine_titandb_num_live_blob_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "live blob file num", - "refId": "A" - }, - { - "datasource": null, - "expr": "sum(tikv_engine_titandb_num_obsolete_blob_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "obsolete blob file num", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob file count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 27 - }, - "id": 3557, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_engine_titandb_live_blob_file_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "live blob file size", - "refId": "A" - }, - { - "datasource": null, - "expr": "sum(tikv_engine_titandb_obsolete_blob_file_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "obsolete blob file size", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob file size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 3523, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_engine_titandb_live_blob_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "live blob size", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Live blob size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The blob cache size.", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 34 - }, - "id": 4655, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "topk(20, avg(tikv_engine_blob_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"}) by(cf, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{cf}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob cache size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The hit rate of block cache", - "fill": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 33 - }, - "id": 4020, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxPerRow": 2, - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_blob_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_cache_hit\"}[2m])) / (sum(rate(tikv_engine_blob_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"$titan_db\", type=\"blob_cache_hit\"}[2m])) + sum(rate(tikv_engine_blob_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"$titan_db\", type=\"blob_cache_miss\"}[2m])))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "all", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob cache hit", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "ops", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 4023, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_blob_iter_touch_blob_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_average\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "A" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_iter_touch_blob_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_percentile95\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "C" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_iter_touch_blob_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_percentile99\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "D" - }, - { - "datasource": null, - "expr": "max(tikv_engine_blob_iter_touch_blob_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_max\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "max", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Iter touched blob file count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 4025, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_engine_titandb_blob_file_discardable_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"}) by (ratio)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ratio}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob file discardable ratio distribution", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 45 - }, - "id": 3414, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_blob_key_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_average\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "A" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_key_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_percentile95\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "C" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_key_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_percentile99\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "D" - }, - { - "datasource": null, - "expr": "max(tikv_engine_blob_key_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_max\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "max", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob key size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 45 - }, - "id": 3446, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_blob_value_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_average\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "avg", - "refId": "A" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_value_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_percentile95\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "B" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_value_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_percentile99\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "C" - }, - { - "datasource": null, - "expr": "max(tikv_engine_blob_value_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_max\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob value size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 51 - }, - "id": 3746, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_blob_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_get\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "get", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob get operations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 51 - }, - "id": 3655, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_blob_get_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "A" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_get_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "B" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_get_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "C" - }, - { - "datasource": null, - "expr": "max(tikv_engine_blob_get_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob get duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 57 - }, - "id": 3338, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_blob_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_seek\"}[2m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "seek", - "refId": "A" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_blob_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_prev\"}[2m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "prev", - "refId": "B" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_engine_blob_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_next\"}[2m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "next", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob iter operations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 57 - }, - "id": 3412, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_blob_seek_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "A" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_seek_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "B" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_seek_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "C" - }, - { - "datasource": null, - "expr": "max(tikv_engine_blob_seek_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob seek duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 63 - }, - "id": 4092, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_blob_next_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "A" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_next_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "B" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_next_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "C" - }, - { - "datasource": null, - "expr": "max(tikv_engine_blob_next_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob next duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 63 - }, - "id": 4093, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_blob_prev_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "A" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_prev_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "B" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_prev_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "C" - }, - { - "datasource": null, - "expr": "max(tikv_engine_blob_prev_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob prev duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 69 - }, - "id": 3645, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_blob_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\"keys.*\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob keys flow", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 69 - }, - "id": 3643, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_blob_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\"bytes.*\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob bytes flow", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 75 - }, - "id": 3657, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_blob_file_read_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_average\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "avg", - "refId": "A" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_file_read_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_percentile99\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "B" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_file_read_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_percentile95\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "C" - }, - { - "datasource": null, - "expr": "max(tikv_engine_blob_file_read_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_max\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob file read duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 75 - }, - "id": 3408, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_average\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "avg", - "refId": "A" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_percentile99\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "B" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_file_write_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_percentile95\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "C" - }, - { - "datasource": null, - "expr": "max(tikv_engine_blob_file_write_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_max\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob file write duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 81 - }, - "id": 3651, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_blob_file_synced{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "sync", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob file sync operations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 81 - }, - "id": 3653, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_blob_file_sync_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "A" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_file_sync_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_percentile95\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "B" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_file_sync_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_percentile99\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "C" - }, - { - "datasource": null, - "expr": "max(tikv_engine_blob_file_sync_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_max\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob file sync duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 87 - }, - "id": 5018, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_blob_gc_action_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"}[2m])) by (type)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob GC action", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 87 - }, - "id": 3410, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_blob_gc_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "A" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_gc_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_percentile95\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "B" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_gc_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_percentile99\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "C" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_gc_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_max\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob GC duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 93 - }, - "id": 3649, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\"keys.*\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob GC keys flow", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "show": true - }, - { - "format": "decbytes", - "label": "", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 93 - }, - "id": 3340, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_blob_gc_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\"bytes.*\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob GC bytes flow", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "show": true - }, - { - "format": "decbytes", - "label": "", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 99 - }, - "id": 4021, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_blob_gc_input_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "A" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_gc_input_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_percentile95\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "B" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_gc_input_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_percentile99\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "C" - }, - { - "datasource": null, - "expr": "max(tikv_engine_blob_gc_input_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_max\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob GC input file size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 99 - }, - "id": 4022, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_engine_blob_gc_output_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "A" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_gc_output_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_percentile95\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "B" - }, - { - "datasource": null, - "expr": "avg(tikv_engine_blob_gc_output_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_percentile99\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "C" - }, - { - "datasource": null, - "expr": "max(tikv_engine_blob_gc_outputt_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_max\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob GC output file size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 105 - }, - "id": 3344, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_engine_blob_gc_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"}[2m])) by (type)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Blob GC file count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "repeat": "titan_db", - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Titan - $titan_db", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 37 - }, - "id": 2820, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 95 - }, - "hiddenSeries": false, - "id": 2991, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"waiter_manager.*\"}[2m])) by (instance, name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{name}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"deadlock_detect.*\"}[2m])) by (instance, name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{name}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Lock Manager Thread CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 95 - }, - "hiddenSeries": false, - "id": 2877, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_lock_manager_task_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Lock Manager Handled tasks", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 103 - }, - "hiddenSeries": false, - "id": 2993, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_lock_manager_waiter_lifetime_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_lock_manager_waiter_lifetime_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_lock_manager_waiter_lifetime_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99%", - "refId": "B" - }, - { - "datasource": null, - "expr": "histogram_quantile(0.9999, sum(rate(tikv_lock_manager_waiter_lifetime_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "99.99%", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Waiter lifetime duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 103 - }, - "hiddenSeries": false, - "id": 4018, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(max_over_time(tikv_lock_manager_wait_table_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[15s])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(max_over_time(tikv_lock_wait_queue_entries_gauge_vec{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[15s])) by (type)", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Lock Waiting Queue", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 111 - }, - "id": 2995, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_lock_manager_detect_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_lock_manager_detect_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "A" - }, - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_lock_manager_detect_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Deadlock detect duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 2, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 111 - }, - "id": 2934, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_lock_manager_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Detect error", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 119 - }, - "id": 4019, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pluginVersion": "6.1.6", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(max_over_time(tikv_lock_manager_detector_leader_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[15s])) by (instance)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Deadlock detector leader", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "none", - "label": "", - "logBase": 1, - "max": "2", - "min": "0", - "show": false - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 119 - }, - "hiddenSeries": false, - "id": 23763572093, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "tikv_pessimistic_lock_memory_size{tidb_cluster=\"$tidb_cluster\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total pessimistic locks memory size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 127 - }, - "hiddenSeries": false, - "id": 23763572094, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_in_memory_pessimistic_locking{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{result}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "In-memory pessimistic locking result", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of active keys and waiters.", - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 127 - }, - "hiddenSeries": false, - "id": 23763573091, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(tikv_lock_wait_queue_entries_gauge_vec{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Pessimistic lock activities", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The length includes the entering transaction itself", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 135 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 23763573092, - "legend": { - "show": false - }, - "links": [], - "pluginVersion": "7.5.11", - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(increase(tikv_lock_wait_queue_length_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A", - "step": 4 - } - ], - "title": "Lengths of lock wait queues when transaction enqueues", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "format": "short", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Pessimistic Locking", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 38 - }, - "id": 8389, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe CPU utilization of resolved ts worker", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 38 - }, - "hiddenSeries": false, - "id": 8385, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"resolved_ts.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Resolved TS Worker CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe CPU utilization of advance ts worker", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 38 - }, - "hiddenSeries": false, - "id": 9162, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"advance_ts.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-tso", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Advance ts Worker CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe CPU utilization of scan lock worker", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 38 - }, - "hiddenSeries": false, - "id": 9164, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"inc_scan.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-scan", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scan lock Worker CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The gap between resolved tso and current time", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 46 - }, - "hiddenSeries": false, - "id": 8387, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_resolved_ts_min_resolved_ts_gap_millis{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 60 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Max Resolved TS gap", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The gap between resolved tso of leaders and current time", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 46 - }, - "hiddenSeries": false, - "id": 23763572077, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(tikv_resolved_ts_min_leader_resolved_ts_gap_millis{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 60 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Max Leader Resolved TS gap", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The region that has minimal resolved ts", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 54 - }, - "hiddenSeries": false, - "id": 23763572078, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(tikv_resolved_ts_min_resolved_ts_region{tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Min Resolved TS Region", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "none", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The region that its leader has minimal resolved ts.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 54 - }, - "hiddenSeries": false, - "id": 23763572079, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(tikv_resolved_ts_min_leader_resolved_ts_region{tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Min Leader Resolved TS Region", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "none", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The time consumed when handle a check leader request", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 62 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 9168, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_resolved_ts_check_leader_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "title": "Check leader duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The status of resolved-ts observe regions", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 62 - }, - "hiddenSeries": false, - "id": 8377, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_resolved_ts_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Observe region status", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Bucketed histogram of region count in a check leader request", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 70 - }, - "hiddenSeries": false, - "id": 12308, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_check_leader_request_item_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_snapshot_size_bucket", - "refId": "A", - "step": 40 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% CheckLeader request region count", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of fail to advance resolved-ts", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 70 - }, - "hiddenSeries": false, - "id": 9166, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(delta(tikv_resolved_ts_fail_advance_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{reason}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Fail advance ts count", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "Total bytes in memory of resolved-ts observe regions's lock heap", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 78 - }, - "hiddenSeries": false, - "id": 8379, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_resolved_ts_lock_heap_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Lock heap size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Bucketed histogram of the check leader request size", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 78 - }, - "hiddenSeries": false, - "id": 8383, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_check_leader_request_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_snapshot_size_bucket", - "refId": "A", - "step": 40 - }, - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_check_leader_request_item_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "{{instance}}-check-num", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "99% CheckLeader request size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "Total bytes of pending commands in the channel", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 86 - }, - "hiddenSeries": false, - "id": 8381, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "avg(tikv_resolved_ts_channel_penging_cmd_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Pending command size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Resolved-TS", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 39 - }, - "id": 2763, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 505 - }, - "id": 2696, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "tikv_allocator_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "thresholds": [], - "title": "Allocator Stats", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Memory", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 3922, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 65 - }, - "hiddenSeries": false, - "id": 3924, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/backup-auto-throttle/", - "fill": 5, - "fillGradient": 2, - "linewidth": 0 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"b.*k.*w.*k.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "backup-{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"backup_io\"}[2m])) by (instance)", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 2, - "legendFormat": "backup-io-{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "B", - "step": 4 - }, - { - "datasource": null, - "exemplar": true, - "expr": "tikv_backup_softlimit{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "backup-auto-throttle-{{instance}}", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Backup CPU Utilization", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fontSize": "100%", - "gridPos": { - "h": 7, - "w": 4, - "x": 12, - "y": 65 - }, - "id": 3926, - "links": [], - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "alias": "", - "align": "auto", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "datasource": null, - "expr": "sum(tikv_backup_thread_pool_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Backup Thread Count", - "transform": "timeseries_aggregations", - "type": "table" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "hiddenSeries": false, - "id": 23763571993, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_cloud_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (cloud, req)", - "interval": "", - "legendFormat": "{{cloud}}-{{req}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "cloud request", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 65 - }, - "hiddenSeries": false, - "id": 5264, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "delta(tikv_backup_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{error}}", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Backup Errors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 72 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 3927, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "max(rate(tikv_backup_range_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", cf=\"write\"}[2m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Backup Write CF SST Size", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "bytes", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 72 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 5266, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "max(rate(tikv_backup_range_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", cf=\"default\"}[2m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Backup Default CF SST Size", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "bytes", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 72 - }, - "hiddenSeries": false, - "id": 3929, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_backup_range_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "total", - "metric": "", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "rate(tikv_backup_range_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}} {{cf}}", - "metric": "", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Backup SST Generation Throughput", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "show": true - }, - { - "format": "Bps", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 0, - "y": 79 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 5597, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Backup Scan SST Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 4, - "y": 79 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 3931, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan\"}[2m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Backup Scan SST Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 10, - "y": 79 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 6905, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"save.*\"}[2m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Backup Save SST Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 79 - }, - "hiddenSeries": false, - "id": 3928, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}} - 99%", - "metric": "", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "histogram_quantile(0.95, sum(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}} - 95%", - "refId": "B", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_backup_range_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_backup_range_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}} - avg", - "refId": "C", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Backup SST Duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 86 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 3930, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "max(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "External Storage Create Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 86 - }, - "id": 4936, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(1, sum(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,type))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}-100%", - "refId": "E" - }, - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,type))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}-99%", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "External Storage Create Duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 1, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 93 - }, - "id": 5267, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"analyze.*|checksum.*\"}[2m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-100%", - "refId": "E" - }, - { - "datasource": null, - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"analyze.*|checksum.*\"}[2m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-99%", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Checksum Request Duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 1, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 93 - }, - "id": 5269, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{device}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "IO Utilization", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 100 - }, - "id": 5925, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/import-count.*/", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "import-{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance, tid) > 0", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "backup-{{instance}}-{{tid}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "C", - "step": 4 - }, - { - "datasource": null, - "expr": "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "import-count-{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "D", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Import CPU Utilization", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fontSize": "100%", - "gridPos": { - "h": 7, - "w": 4, - "x": 12, - "y": 100 - }, - "id": 5926, - "links": [], - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "alias": "", - "align": "auto", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "datasource": null, - "expr": "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Import Thread Count", - "transform": "timeseries_aggregations", - "type": "table" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 100 - }, - "id": 5932, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "delta(tikv_import_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{type}} {{error}} {{instance}}", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Import Errors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 107 - }, - "id": 5931, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(0.99, avg(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, request))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{request}}-99%", - "refId": "A" - }, - { - "datasource": null, - "expr": "histogram_quantile(0.5, sum(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, request, instance))", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{request}}-50%", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Import RPC Duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 107 - }, - "id": 6267, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", request!=\"switch_mode\"}[2m])) by (instance, request)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}} - {{request}}", - "metric": "tikv_grpc_msg_duration_seconds_bucket", - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "expr": "sum(rate(tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", request!=\"switch_mode\"}[2m])) by (request)", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "total - {{request}}", - "metric": "tikv_grpc_msg_duration_seconds_bucket", - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Import RPC Ops", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 114 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 5930, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "max(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", request=~\"download|write\"}[2m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Import Write/Download RPC Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 114 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 5929, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"queue\"}[2m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Import Wait Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 114 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 6906, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"read\"}[2m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Import Read SST Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 114 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 5928, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"rewrite\"}[2m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Import Rewrite SST Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 121 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 5939, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "max(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", request=~\"ingest\"}[2m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Import Ingest RPC Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 121 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 5938, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "max(rate(tikv_import_ingest_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ingest\"}[2m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Import Ingest SST Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 121 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 5937, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "max(rate(tikv_import_ingest_byte{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Import Ingest SST Bytes", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 121 - }, - "id": 5927, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_import_download_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_import_download_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "total", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Import Download SST Throughput", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "show": true - }, - { - "format": "Bps", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 128 - }, - "id": 12309, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "delta(tikv_import_local_write_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{type}} {{instance}}", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Import Local Write keys", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 128 - }, - "id": 12310, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "delta(tikv_import_local_write_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{type}} {{instance}}", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Import Local Write bytes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The accumulated TTL expired KV count during backup", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 135 - }, - "hiddenSeries": false, - "id": 23763572861, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(tikv_backup_raw_expired_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "hide": true, - "interval": "", - "legendFormat": "{{instance}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(tikv_backup_raw_expired_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})", - "hide": false, - "interval": "", - "legendFormat": "sum", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TTL Expired", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Backup & Import", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 41 - }, - "id": 4466, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Total number of encryption data keys in use", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 4464, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "tikv_encryption_data_key_storage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Encryption data keys", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - }, - { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Number of files being encrypted", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 4554, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "tikv_encryption_file_num{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Encrypted files", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Flag to indicate if encryption is initialized", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 66 - }, - "id": 4555, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "tikv_encryption_is_initialized{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Encryption initialized", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - }, - { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Total size of encryption meta files", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 66 - }, - "id": 4556, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "tikv_encryption_meta_file_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{name}}-{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Encryption meta files size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 74 - }, - "id": 4557, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "sum(rate(tikv_coprocessor_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\" ,metric=\"encrypt_data_nanos\"}[2m])) by (req)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "encrypt-{{req}}", - "refId": "A" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_coprocessor_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\" ,metric=\"decrypt_data_nanos\"}[2m])) by (req)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "decrypt-{{req}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Encrypt/decrypt data nanos", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Writing or reading file duration (second)", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 74 - }, - "id": 4559, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "expr": "histogram_quantile(1, sum(rate(tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation))", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "max-{{type}}-{{operation}}", - "refId": "A" - }, - { - "datasource": null, - "expr": "histogram_quantile(0.95, sum(rate(tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "95%-{{type}}-{{operation}}", - "refId": "B" - }, - { - "datasource": null, - "expr": "sum(rate(tikv_encryption_write_read_file_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation) / sum(rate(tikv_encryption_write_read_file_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "avg-{{type}}-{{operation}}", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Read/write encryption meta duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Encryption", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 42 - }, - "id": 13016, - "panels": [ - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "text": "Disabled" - }, - "1": { - "text": "Enabled" - } - }, - "type": "value" - } - ], - "noValue": "Disabled", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(0, 0, 0, 0.2)" - }, - { - "color": "dark-red", - "value": 0 - }, - { - "color": "dark-green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 0, - "y": 55 - }, - "id": 14361, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "7.5.11", - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "tikv_log_backup_enabled{instance=~\"$instance\"}", - "instant": true, - "interval": "", - "legendFormat": "{{ instance }}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "Endpoint Status", - "transformations": [], - "type": "stat" - }, - { - "datasource": null, - "description": "The average flush size of last 30mins.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 5, - "y": 55 - }, - "id": 14507, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "7.5.11", - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "increase(tikv_log_backup_flush_file_size_sum{instance=~\"$instance\"}[30m]) / on(instance) increase(tikv_log_backup_flush_duration_sec_count{stage=~\"save_files\",instance=~\"$instance\"}[30m])", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "{{ instance }}", - "refId": "B" - } - ], - "title": "Average Flush Size ", - "type": "stat" - }, - { - "datasource": null, - "description": "The current total flushed file number of this run.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 13, - "y": 55 - }, - "id": 14363, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "7.5.11", - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "round(increase(tikv_log_backup_flush_file_size_count{instance=~\"$instance\"}[30m]))", - "instant": true, - "interval": "", - "legendFormat": "{{ instance }}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "Flushed Files (Last 30m) Per Host", - "type": "stat" - }, - { - "datasource": null, - "description": "This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot. \n**NOTE: The size may get reduced if some of TiKVs reboot.**", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 21, - "y": 55 - }, - "id": 14508, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "7.5.11", - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "round(sum(increase(tikv_log_backup_flush_duration_sec_count{stage=~\"save_files\",instance=~\"$instance\"}[30m])))", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "{{ instance }}", - "refId": "B" - } - ], - "title": "Flush Times (Last 30m)", - "type": "stat" - }, - { - "datasource": null, - "description": "This is the summary of the size has been flushed, summered by the data each TiKV has flushed since last boot. \n**NOTE: The size may get reduced if some of TiKVs reboot.**", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 21, - "y": 57 - }, - "id": 14362, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "7.5.11", - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(increase(tikv_log_backup_flush_file_size_sum{instance=~\"$instance\"}[30m]))", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "{{ instance }}", - "refId": "B" - } - ], - "title": "Total Flushed Size (Last 30m)", - "type": "stat" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "text": "Running" - }, - "1": { - "text": "Paused" - }, - "2": { - "text": "Error" - } - }, - "type": "value" - } - ], - "noValue": "Disabled", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(0, 0, 0, 0.2)" - }, - { - "color": "dark-green", - "value": 0 - }, - { - "color": "#EAB839", - "value": 1 - }, - { - "color": "dark-red", - "value": 2 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 0, - "y": 59 - }, - "id": 14907, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "7.5.11", - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "min(tikv_log_backup_task_status{instance=~\"$instance\"})", - "instant": true, - "interval": "", - "legendFormat": "{{ instance }}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "Task Status", - "transformations": [], - "type": "stat" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "min": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-blue" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 2, - "y": 59 - }, - "id": 15361, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "name" - }, - "pluginVersion": "7.5.11", - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "tidb_log_backup_advancer_owner > 0", - "instant": true, - "interval": "", - "legendFormat": "{{ instance }}", - "refId": "A" - } - ], - "title": "Advancer Owner", - "type": "stat" - }, - { - "datasource": null, - "description": "This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot. \n**NOTE: The size may get reduced if some of TiKVs reboot.**", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 21, - "y": 60 - }, - "id": 14911, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "7.5.11", - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "round(sum(increase(tikv_log_backup_flush_file_size_count{instance=~\"$instance\"}[30m])))", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "{{ instance }}", - "refId": "B" - } - ], - "title": "Flush Files (Last 30m)", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of log backup threads. \n**(Note this is the average usage for a period of time, some peak of CPU usage may be lost.)**", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 10, - "w": 6, - "x": 0, - "y": 63 - }, - "hiddenSeries": false, - "id": 13262, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"backup_stream|log-backup-scan(-[0-9]+)?\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU Usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:646", - "format": "percentunit", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:647", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 6, - "x": 6, - "y": 63 - }, - "hiddenSeries": false, - "id": 12843, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "rate(tikv_log_backup_handle_kv_batch_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Handle Event Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:563", - "format": "ops", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:564", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The data rate of initial scanning emitting events.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 6, - "x": 12, - "y": 63 - }, - "hiddenSeries": false, - "id": 14135, - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "rate(tikv_log_backup_incremental_scan_bytes_sum{instance=~\"$instance\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Initial Scan Generate Event Throughput", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:136", - "format": "binBps", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:137", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "alert": { - "alertRuleTags": {}, - "conditions": [ - { - "evaluator": { - "params": [ - 600000 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "5m", - "frequency": "1m", - "handler": 1, - "name": "Checkpoint Lag Too Huge", - "noDataState": "no_data", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "unit": "ms" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 6, - "x": 18, - "y": 63 - }, - "hiddenSeries": false, - "id": 14774, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "time() * 1000 - max(tidb_log_backup_last_checkpoint / 262144 > 0) by (task)", - "instant": false, - "interval": "", - "legendFormat": "{{ task }}", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "time() * 1000", - "hide": true, - "interval": "", - "legendFormat": "Current Time", - "refId": "B" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 600000, - "visible": true - } - ], - "timeRegions": [], - "title": "Abnormal Checkpoint TS Lag", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:228", - "format": "ms", - "logBase": 1, - "max": "3000000", - "min": "0", - "show": true - }, - { - "$$hashKey": "object:229", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The estimated memory usage by the streaming backup module.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 6, - "x": 0, - "y": 73 - }, - "hiddenSeries": false, - "id": 13100, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "tikv_log_backup_heap_memory{instance=~\"$instance\"}", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory Of Events", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:563", - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:564", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 6, - "x": 6, - "y": 73 - }, - "hiddenSeries": false, - "id": 14630, - "legend": { - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "tikv_log_backup_observed_region{instance=~\"$instance\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "sum(tikv_log_backup_observed_region{instance=~\"$instance\"})", - "hide": false, - "interval": "", - "legendFormat": "total", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Observed Region Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:136", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:137", - "format": "none", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The errors met when backing up.\n**They are retryable, don't worry.**", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 73 - }, - "hiddenSeries": false, - "id": 13101, - "legend": { - "avg": false, - "current": false, - "hideZero": true, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": true, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "increase(tikv_log_backup_errors{instance=~\"$instance\"}[$__interval])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{type}}@{{instance}}", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "tikv_log_backup_errors{instance=~\"$instance\"}", - "hide": true, - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Errors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:563", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:564", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 6, - "x": 18, - "y": 73 - }, - "hiddenSeries": false, - "id": 14910, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:240", - "alias": "Current Time", - "dashes": true, - "fill": 0 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "max(tidb_log_backup_last_checkpoint{instance=~\"$instance\"} / 262144 > 0) by (task)", - "instant": false, - "interval": "", - "legendFormat": "{{ task }}", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "time() * 1000", - "hide": false, - "interval": "", - "legendFormat": "Current Time", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Checkpoint TS of Tasks", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:169", - "format": "dateTimeAsIsoNoDateIfToday", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:170", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The errors met when backing up.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 78 - }, - "hiddenSeries": false, - "id": 14908, - "legend": { - "avg": false, - "current": false, - "hideZero": true, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": true, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "increase(tikv_log_backup_fatal_errors{instance=~\"$instance\"}[$__interval])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{type}}@{{instance}}", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "", - "hide": true, - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "", - "refId": "B" - } - ], - "thresholds": [ - { - "$$hashKey": "object:3232", - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0, - "yaxis": "left" - } - ], - "timeRegions": [], - "title": "Fatal Errors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:563", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:564", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateBlues", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The duration of flushing a batch of file.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 83 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 14078, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(increase(tikv_log_backup_flush_duration_sec_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"save_files\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Flush Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateReds", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The duration of scanning the initial data from local DB and transform them into apply events. \n", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 83 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 14136, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(increase(tikv_log_backup_initial_scan_duration_sec_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Initial scanning duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateGreens", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The duration of converting a raft request into a apply event. \n*This duration is for consuming a batch of events.*", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 83 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 13934, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(increase(tikv_log_backup_event_handle_duration_sec_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"to_stream_event\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Convert Raft Event duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateGreens", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The duration of waiting the mutex of the controller. \n*This duration is for consuming a batch of events.*", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 83 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 12840, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(increase(tikv_log_backup_event_handle_duration_sec_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"get_router_lock\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Wait for Lock Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateCividis", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The number of KV-modify of each raft command observed.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 90 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 15059, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(increase(tikv_log_backup_handle_kv_batch_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Command Batch Size", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "short", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The total cost of saving an event into temporary file. \n*This duration is for consuming a batch of events.*", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 90 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 12841, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(increase(tikv_log_backup_event_handle_duration_sec_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"save_to_temp_file\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Save to Temp File Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The total cost of writing a event into temporary file.\nComparing to the ***Save*** duration, it doesn't contain the time cost of routing the task by range / task. \n*This duration is for consuming a batch of events, for one region or one table.*", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 90 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 13552, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(increase(tikv_log_backup_on_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", stage=\"write_to_tempfile\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Write to Temp File Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "The duration of collecting metadata and call the UNIX system call *write* for each event. \n*This duration is for consuming a batch of events, for one region or one table.*", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 90 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 13551, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(increase(tikv_log_backup_on_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", stage=\"syscall_write\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "System Write Call Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The internal message type count.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 97 - }, - "hiddenSeries": false, - "id": 14914, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_log_backup_interal_actor_acting_duration_sec_count{instance=~\"$instance\"}[$__rate_interval])) by (message)", - "interval": "", - "legendFormat": "{{ message }}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Internal Message Type", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:103", - "format": "ops", - "logBase": 2, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:104", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "watch_task": "orange" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The internal handling message duration.", - "fieldConfig": { - "defaults": { - "unit": "s" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 97 - }, - "hiddenSeries": false, - "id": 14912, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(histogram_quantile(0.99, rate(tikv_log_backup_interal_actor_acting_duration_sec_bucket{instance=~\"$instance\"}[10m]))) by (message)", - "interval": "", - "legendFormat": "{{ message }}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Internal Message Handling Duration (P99)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:103", - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:104", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The internal handling message duration.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 97 - }, - "hiddenSeries": false, - "id": 14913, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(histogram_quantile(0.9, rate(tikv_log_backup_interal_actor_acting_duration_sec_bucket{instance=~\"$instance\"}[10m]))) by (message)", - "interval": "", - "legendFormat": "{{ message }}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Internal Message Handling Duration (P90)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:103", - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:104", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The internal read throughput of RocksDB during initial scanning. This panel can roughly present the read through to the hard disk of initial scanning.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 103 - }, - "hiddenSeries": false, - "id": 14271, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_log_backup_initial_scan_operations{instance=~\"$instance\", op=~\"read_bytes\"}[$__rate_interval])) BY (op, cf)", - "interval": "", - "legendFormat": "{{ cf }}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Initial Scan RocksDB Throughput ", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:103", - "format": "binBps", - "logBase": 2, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:104", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Misc statistics of RocksDB during initial scanning.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 103 - }, - "hiddenSeries": false, - "id": 14270, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_log_backup_initial_scan_operations{instance=~\"$instance\", op!~\"read_bytes\"}[$__rate_interval])) BY (op, cf) > 0", - "interval": "", - "legendFormat": "{{ cf }}/{{ op }}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Initial Scan RocksDB Operation ", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:103", - "format": "ops", - "logBase": 2, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:104", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "leader-changed": "blue", - "region-changed": "purple" - }, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The reason of triggering initial scanning.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 103 - }, - "hiddenSeries": false, - "id": 14915, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": false - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(increase(tikv_log_backup_initial_scan_reason{instance=~\"$instance\"}[$__rate_interval])) by (reason)", - "interval": "", - "legendFormat": "{{ message }}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Initial Scanning Trigger Reason", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:2608", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:2609", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "del": "dark-red", - "put": "green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 103 - }, - "hiddenSeries": false, - "id": 15176, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(rate(tikv_log_backup_metadata_key_operation{instance=~\"$instance\"}[$__rate_interval])) by (type)", - "interval": "", - "legendFormat": "{{ type }}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Region Checkpoint Key Putting", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:2608", - "format": "cps", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:2609", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 109 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 15544, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(increase(tidb_log_backup_advancer_batch_size_bucket{type=\"checkpoint\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Request Checkpoint Batch Size", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "none", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 109 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 15716, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(increase(tidb_log_backup_advancer_tick_duration_sec_bucket{tidb_cluster=\"$tidb_cluster\", step=~\"tick\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Tick Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "upper" - }, - { - "aliasColors": { - "epoch-not-match": "purple", - "not-leader": "blue", - "watch_task": "orange" - }, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The reason of advancer failed to be advanced.", - "fieldConfig": { - "defaults": { - "unit": "none" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 109 - }, - "hiddenSeries": false, - "id": 23763572666, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(increase(tidb_log_backup_region_request_failure{reason!=\"retryable-scan-region\"}[$__interval])) by (reason)", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ reason }}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "", - "hide": false, - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Region Checkpoint Failure Reason", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:103", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:104", - "format": "none", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "fail": "red", - "success": "green", - "watch_task": "orange" - }, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The result of getting region checkpoints.", - "fieldConfig": { - "defaults": { - "unit": "none" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 109 - }, - "hiddenSeries": false, - "id": 23763572665, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:834", - "alias": "fail", - "transform": "negative-Y", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(increase(tidb_log_backup_region_request[$__interval])) by (result)", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ result }}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": null, - "exemplar": true, - "expr": "", - "hide": false, - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Request Result", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:103", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:104", - "format": "none", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "watch_task": "orange" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The internal handling message duration.", - "fieldConfig": { - "defaults": { - "unit": "s" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 116 - }, - "hiddenSeries": false, - "id": 15359, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:1017", - "alias": "consistency-check", - "yaxis": 1 - }, - { - "$$hashKey": "object:1018", - "alias": "get-checkpoints-of-store", - "yaxis": 2 - }, - { - "$$hashKey": "object:1019", - "alias": "get-checkpoints-in-range", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(histogram_quantile(0.99, rate(tidb_log_backup_advancer_tick_duration_sec_bucket[10m]))) by (step)", - "interval": "", - "legendFormat": "{{ step }}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Tick Duration (P99)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:103", - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:104", - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "watch_task": "orange" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The internal handling message duration.", - "fieldConfig": { - "defaults": { - "unit": "s" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 116 - }, - "hiddenSeries": false, - "id": 15360, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:1091", - "alias": "get-checkpoints-of-store", - "yaxis": 2 - }, - { - "$$hashKey": "object:1092", - "alias": "get-checkpoints-in-range", - "yaxis": 2 - }, - { - "$$hashKey": "object:1093", - "alias": "consistency-check", - "yaxis": 1 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "sum(histogram_quantile(0.9, rate(tidb_log_backup_advancer_tick_duration_sec_bucket[10m]))) by (step)", - "interval": "", - "legendFormat": "{{ step }}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Tick Duration (P90)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:103", - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:104", - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "watch_task": "orange" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The frequent of getting region level checkpoint.", - "fieldConfig": { - "defaults": { - "unit": "none" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 116 - }, - "hiddenSeries": false, - "id": 23763572733, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:1091", - "alias": "get-checkpoints-of-store", - "yaxis": 2 - }, - { - "$$hashKey": "object:1092", - "alias": "get-checkpoints-in-range", - "yaxis": 2 - }, - { - "$$hashKey": "object:1093", - "alias": "consistency-check", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "rate(tidb_log_backup_advancer_tick_duration_sec_count{step=\"get-regions-in-range\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{ step }} {{ instance }}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Get Region Operation Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:103", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:104", - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "watch_task": "orange" - }, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "The variant of checkpoint group.", - "fieldConfig": { - "defaults": { - "unit": "none" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 116 - }, - "hiddenSeries": false, - "id": 23763572734, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:1091", - "alias": "get-checkpoints-of-store", - "yaxis": 2 - }, - { - "$$hashKey": "object:1092", - "alias": "get-checkpoints-in-range", - "yaxis": 2 - }, - { - "$$hashKey": "object:1093", - "alias": "consistency-check", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "exemplar": true, - "expr": "increase(tidb_log_backup_advancer_tick_duration_sec_count{step=\"try-advance\"}[$__interval])", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ step }} {{ instance }}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Try Advance Trigger Time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:103", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:104", - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": null, - "refId": "A" - } - ], - "title": "Backup Log", - "type": "row" - } - ], - "refresh": "1m", - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": null, - "definition": "", - "hide": 2, - "includeAll": false, - "label": "K8s-cluster", - "multi": false, - "name": "k8s_cluster", - "options": [], - "query": { - "query": "label_values(tikv_engine_block_cache_size_bytes, k8s_cluster)", - "refId": "quota-k8s_cluster-Variable-Query" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": null, - "definition": "", - "hide": 2, - "includeAll": false, - "label": "tidb_cluster", - "multi": false, - "name": "tidb_cluster", - "options": [], - "query": { - "query": "label_values(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", - "refId": "quota-tidb_cluster-Variable-Query" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": null, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "db", - "multi": true, - "name": "db", - "options": [], - "query": { - "query": "label_values(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, db)", - "refId": "quota-db-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": null, - "definition": "label_values(tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, type)", - "hide": 0, - "includeAll": true, - "label": "command", - "multi": true, - "name": "command", - "options": [], - "query": { - "query": "query_result(tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"} != 0)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "/type=\"([^\"]+)\"/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": null, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Instance", - "multi": false, - "name": "instance", - "options": [], - "query": { - "query": "label_values(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", - "refId": "quota-instance-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": null, - "definition": "label_values(tikv_engine_titandb_num_live_blob_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, db)", - "hide": 2, - "includeAll": true, - "label": "titan_db", - "multi": true, - "name": "titan_db", - "options": [], - "query": { - "query": "label_values(tikv_engine_titandb_num_live_blob_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, db)", - "refId": "quota-titan_db-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "TiKV-Details", - "uid": "RDVQiEzZz", - "version": 1, - "weekStart": "" - }, - "overwrite": false -} \ No newline at end of file diff --git a/rootfs/usr/share/grafana/api/dashboards/tikv_summary.json b/rootfs/usr/share/grafana/api/dashboards/tikv_summary.json deleted file mode 100644 index 28da7ea..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/tikv_summary.json +++ /dev/null @@ -1,4926 +0,0 @@ -{ - "dashboard":{ - "__inputs": [ - { - "name": "DS_TEST-CLUSTER", - "label": "test-cluster", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "6.1.6" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": null, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "id": null, - "iteration": 1566459338986, - "links": [], - "panels": [ - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2742, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The storage size per TiKV instance", - "editable": true, - "error": false, - "fill": 5, - "grid": {}, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 1 - }, - "id": 56, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"used\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Store size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The available capacity size of each TiKV instance", - "editable": true, - "error": false, - "fill": 5, - "grid": {}, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 1 - }, - "id": 1706, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"available\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Available size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The capacity size per TiKV instance", - "editable": true, - "error": false, - "fill": 5, - "grid": {}, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 1 - }, - "id": 1707, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"capacity\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Capacity size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe CPU usage of each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 1708, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", app_kubernetes_io_component=\"tikv\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe memory usage of each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 1709, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", app_kubernetes_io_component=\"tikv\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe I/O utilization per TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 1710, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{device}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "IO utilization", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total bytes of read and write in each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 17 - }, - "id": 1711, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"wal_file_bytes\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}-write", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=~\"bytes_read|iter_bytes_read\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}-read", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "MBps", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The QPS of different kinds of commands in each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 25 - }, - "id": 1713, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (instance,type)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{type}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "QPS", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total number of the gRPC message failures", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 25 - }, - "id": 1712, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}-grpc-msg-fail", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(delta(tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"noop\"}[2m])) by (instance) < 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-pd-heartbeat", - "refId": "B" - }, - { - "expr": "sum(rate(tikv_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{type}}", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Errps", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of leaders per TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 1715, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - }, - { - "expr": "delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m]) < -10", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Leader", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe number of Regions on each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 33 - }, - "id": 1714, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Region", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "Cluster", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 2743, - "panels": [ - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 0 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "frequency": "60s", - "handler": 1, - "name": "Critical error alert", - "noDataState": "no_data", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 2 - }, - "id": 2741, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{type}}", - "refId": "A" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Critical error", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "It contains some kinds of events such as write stall, channel full, scheduler busy, and coprocessor full, which will make the TiKV instance unavailable temporarily.", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 1584, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "scheduler-{{instance}}", - "metric": "", - "refId": "A", - "step": 4 - }, - { - "expr": "sum(rate(tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "channelfull-{{instance}}-{{type}}", - "metric": "", - "refId": "B", - "step": 4 - }, - { - "expr": "sum(rate(tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type='full'}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "coprocessor-{{instance}}", - "metric": "", - "refId": "C", - "step": 4 - }, - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "stall-{{instance}}", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Server is busy", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 0 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "10s", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "frequency": "10s", - "handler": 1, - "message": "TiKV server report failures", - "name": "server report failures alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total number of reporting failure messages", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 18, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_server_report_failure_msg_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type,instance,store_id)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{type}} - to - {{store_id}}", - "metric": "tikv_server_raft_store_msg_total", - "refId": "A", - "step": 10 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Server report failures", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of Raftstore errors per type on each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 1718, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", status!~\"success|all\"}[2m])) by (instance, status)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{status}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Raftstore error", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of different scheduler errors on each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 1719, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"snapshot_err|prepare_write_err\"}[2m])) by (instance, stage)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{stage}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Scheduler error", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of different coprocessor errors on each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 1720, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{reason}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Coprocessor error", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The number of different gRPC message errors on each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 23 - }, - "id": 1721, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{type}}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "gRPC message error", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of dropped leader in each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 30 - }, - "id": 1722, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Leader drop", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of missing leaders per TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 30 - }, - "id": 1723, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(tikv_raftstore_leader_missing{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Leader missing", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "Errors", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 2 - }, - "id": 2744, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe total size of each column family", - "editable": true, - "error": false, - "fill": 3, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 3 - }, - "id": 33, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CF size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The storage size per TiKV instance", - "editable": true, - "error": false, - "fill": 5, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 3 - }, - "id": 1705, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"used\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Store size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 0 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "datasourceId": 1, - "model": { - "expr": "sum(rate(tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{type}}", - "metric": "", - "refId": "A", - "step": 10 - }, - "params": [ - "A", - "10s", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "frequency": "10s", - "handler": 1, - "message": "TiKV channel full", - "name": "TiKV channel full alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total number of channel full errors on each TiKV instance", - "editable": true, - "error": false, - "fill": 3, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 22, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{type}}", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Channel full", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 1073741824 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "B", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "frequency": "60s", - "handler": 1, - "name": "approximate region size alert", - "noDataState": "no_data", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The approximate Region size", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 1481, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_raftstore_region_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_region_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Approximate Region size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "Server", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 2745, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of different kinds of gRPC message", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 4 - }, - "id": 95, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_grpc_msg_duration_seconds_bucket", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "gRPC message count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The count of different kinds of gRPC message which is failed", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 4 - }, - "id": 107, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_grpc_msg_fail_total", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "gRPC message failed", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The gRPC message duration per message type (P99)", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 - }, - "id": 98, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (le, type))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "99% gRPC message duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 10, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 - }, - "id": 2532, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_grpc_msg_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Average gRPC message duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 2, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "gRPC", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 2746, - "panels": [ - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 1.7 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "datasourceId": 1, - "model": { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(raftstore|rs)_.*\"}[2m])) by (instance)", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 20 - }, - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0m", - "frequency": "60s", - "handler": 1, - "message": "TiKV raftstore thread CPU usage is high", - "name": "TiKV raft store CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of raftstore thread", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 5 - }, - "id": 61, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(raftstore|rs)_.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.85 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Raft store CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 1.8 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0m", - "frequency": "1m", - "handler": 1, - "message": "TiKV async apply thread CPU usage is high", - "name": "TiKV async apply CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of async apply", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 5 - }, - "id": 79, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"apply_[0-9]+\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.9 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Async apply CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 3.6 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0m", - "frequency": "1m", - "handler": 1, - "message": "TiKV scheduler worker thread CPU usage is high", - "name": "TiKV scheduler worker CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of scheduler worker", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 12 - }, - "id": 64, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sched_.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.9 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Scheduler worker CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 3.6 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0m", - "frequency": "1m", - "handler": 1, - "message": "TiKV gRPC poll thread CPU usage is high", - "name": "TiKV gRPC poll CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of gRPC", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 12 - }, - "id": 105, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"grpc.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.9 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "gRPC poll CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 7.2 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "", - "frequency": "1m", - "handler": 1, - "message": "TiKV Coprocessor thread CPU alert", - "name": "TiKV Coprocessor CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of coprocessor", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 19 - }, - "id": 78, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_normal.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} - normal", - "refId": "A", - "step": 4 - }, - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_high.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} - high", - "refId": "B", - "step": 4 - }, - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_low.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} - low", - "refId": "C", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.9 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Coprocessor CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 3.6 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "", - "frequency": "1m", - "handler": 1, - "message": "TiKV Storage ReadPool thread CPU usage is high", - "name": "TiKV Storage ReadPool CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of readpool", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 19 - }, - "id": 1908, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_norm.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - normal", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - }, - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_high.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - high", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "B", - "step": 4 - }, - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_low.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - low", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "C", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.9 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Storage ReadPool CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe CPU utilization of split check", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 26 - }, - "id": 68, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"split_check\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Split check CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of RocksDB", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 26 - }, - "id": 69, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"rocksdb.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "warning", - "fill": true, - "line": true, - "op": "gt", - "value": 1 - }, - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 4 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "RocksDB CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 2531, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"gc_worker.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "GC worker CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The CPU utilization of snapshot worker", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 33 - }, - "id": 67, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"snapshot_worker\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Snapshot worker CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "Thread CPU", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 2747, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": " \tThe count of requests that TiKV sends to PD", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "id": 1069, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 350, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ type }}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "PD requests", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The time consumed by requests that TiKV sends to PD", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 1070, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 350, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_pd_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ type }}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "PD request duration (average)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total number of PD heartbeat messages", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 14 - }, - "id": 1215, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 350, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ type }}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "PD heartbeats", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "opm", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "The total number of peers validated by the PD worker", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 1396, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 350, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_pd_validate_peer_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ type }}", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "PD validate peers", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "PD", - "type": "row" - } - ], - "refresh": "1m", - "schemaVersion": 18, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": { }, - "datasource": null, - "hide": 2, - "includeAll": false, - "label": "K8s-cluster", - "multi": false, - "name": "k8s_cluster", - "options": [ ], - "query": "label_values(tikv_engine_block_cache_size_bytes, k8s_cluster)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { }, - "datasource": null, - "hide": 2, - "includeAll": false, - "label": "tidb_cluster", - "multi": false, - "name": "tidb_cluster", - "options": [ ], - "query": "label_values(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": null, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "db", - "multi": true, - "name": "db", - "options": [], - "query": "label_values(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, db)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": null, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "command", - "multi": true, - "name": "command", - "options": [], - "query": "label_values(tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, type)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": {}, - "datasource": null, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Instance", - "multi": false, - "name": "instance", - "options": [], - "query": "label_values(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "TiKV-Summary", - "uid": "X7VQmEzZk", - "version": 1 - },"overwrite": false -} diff --git a/rootfs/usr/share/grafana/api/dashboards/tikv_trouble_shooting.json b/rootfs/usr/share/grafana/api/dashboards/tikv_trouble_shooting.json deleted file mode 100644 index 9e34a8a..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/tikv_trouble_shooting.json +++ /dev/null @@ -1,7985 +0,0 @@ -{ - "dashboard": { - "__inputs": [ - { - "name": "DS_TEST-CLUSTER", - "label": "test-cluster", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "6.1.6" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": null, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "id": null, - "iteration": 1568258597359, - "links": [], - "panels": [ - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2796, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "总 CPU 使用率", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 1708, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", app_kubernetes_io_component=\"tikv\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "QPS of the common request from client. If the same type of request is distributed in multiple machines, there may be hot spot.", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 1713, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"coprocessor|kv_get|kv_batch_get|kv_prewrite|kv_commit\"}[2m])) by (instance, type)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{type}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "QPS", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 3.6 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "", - "frequency": "1m", - "handler": 1, - "message": "TiKV Storage ReadPool thread CPU usage is high", - "name": "TiKV Storage ReadPool CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "点查会走 Storage ReadPool", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 1908, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_norm.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - normal", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - }, - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_high.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - high", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "B", - "step": 4 - }, - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_low.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - low", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "C", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.9 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Storage ReadPool CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 7.2 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "", - "frequency": "1m", - "handler": 1, - "message": "TiKV Coprocessor thread CPU usage is high", - "name": "TiKV Coprocessor CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "SQL of non-point-query accesses Coprocessor Pool", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 78, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_normal.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} - normal", - "refId": "A", - "step": 4 - }, - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_high.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} - high", - "refId": "B", - "step": 4 - }, - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_low.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} - low", - "refId": "C", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.9 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Coprocessor CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 3.6 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "", - "frequency": "1m", - "handler": 1, - "message": "TiKV gRPC poll thread CPU usage is high", - "name": "TiKV gRPC poll CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "gRPC CPU usage. if it is full，the grpc-concurrency param of TiKV needs to be adjusted", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 105, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"grpc.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.9 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "gRPC poll CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "I/O util reflects the busy conditions of disk. If it is almost 100%, it implies that disk is busy and delay may be affected seriously, but it doesn't imply that bandwidth bottleneck is reached.", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 17 - }, - "id": 1710, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{device}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "IO utilization", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "Hot Read", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 2797, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 2 - }, - "id": 2763, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", app_kubernetes_io_component=\"tikv\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 2 - }, - "id": 2764, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (instance,type)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{type}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "QPS", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 2765, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{device}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "IO utilization", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 3.6 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "", - "frequency": "1m", - "handler": 1, - "message": "TiKV gRPC poll thread CPU usage is high", - "name": "TiKV gRPC poll CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 2783, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"grpc.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.9 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "gRPC poll CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 1.7 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0", - "frequency": "1m", - "handler": 1, - "message": "TiKV raft store thread CPU is high", - "name": "TiKV raft store CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "Raftstore 线程 CPU 使用率，这个线程池用来写 Raft log。", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 61, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(raftstore|rs)_.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.85 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Raft store CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 1.8 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "", - "frequency": "1m", - "handler": 1, - "message": "TiKV async apply thread CPU is high", - "name": "TiKV async apply CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "Apply 线程 CPU，这个线程池用来将写入应用到 kv engine 中。", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 79, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"apply_[0-9]+\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.9 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Async apply CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 3.6 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "", - "frequency": "1m", - "handler": 1, - "message": "TiKV Scheduler Worker thread CPU is high", - "name": "TiKV Scheduler Worker CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Scheduler worker CPU usage of thread pool. Transactions will be queued in the Scheduler. If it is close to full, it implies that there are too many old versions of MVCC and transaction processing is too slow.", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 26 - }, - "id": 63, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sched_worker.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.9 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Scheduler Worker CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "Hot Write", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 2 - }, - "id": 2798, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 3 - }, - "id": 34, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - }, - { - "expr": "delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[30s]) < -10", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Leader", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "time of Append log. It is too long means that thread of Raftstore is too busy. It may not be able to handle the heartbeat and leader is switched.", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 3 - }, - "id": 2786, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.999999, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": " 99.9999%", - "metric": "", - "refId": "A", - "step": 4 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "95%", - "refId": "B" - }, - { - "expr": "sum(rate(tikv_raftstore_append_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_append_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "C", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Append log duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Propose 如果在 Raftstore 中等得太久，则说明 Raftstore 线程很忙，或者被卡住了。", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 2787, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.999999, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} ", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Propose 99.9999% wait duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 2789, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"write_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"write_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"write_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"write_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Raft RocksDB write duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "label": null, - "logBase": 2, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": null, - "description": "Raftstore 线程需要处理一些周期性任务。如果处理时间过长，则相当于 Raftstore 线程被卡住了。", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 2829, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.999999, sum(rate(tikv_raftstore_event_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance, type))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{type}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 100, - "yaxis": "left" - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Raft event 99.9999% handle duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "s", - "label": null, - "logBase": 2, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "Leader Drop", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 2799, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "`channel full` happens in Raftstore of TiKV when it can't process all incoming messages. After, subsequent messages will be discarded, which may cause TiDB retry or drop the leader.", - "editable": true, - "error": false, - "fill": 3, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 4 - }, - "id": 22, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{type}}", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Channel full", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "description": "RocksDB write stall 说明写入请求被 compaction 或者其他流量控制措施卡住了。", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 4 - }, - "id": 87, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"write_stall_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"write_stall_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"write_stall_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"write_stall_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Raft RocksDB write stall duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "label": null, - "logBase": 10, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 - }, - "id": 39, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": " 99%", - "metric": "", - "refId": "A", - "step": 4 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "B", - "step": 4 - }, - { - "expr": "sum(rate(tikv_raftstore_append_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_append_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "C", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Append log duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 - }, - "id": 40, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} ", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Append log duration per server", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 2819, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "B" - }, - { - "expr": "sum(rate(tikv_raftstore_commit_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_commit_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Commit log duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 2821, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 99%", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Commit log duration per server", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "Channel Full", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 2800, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 5 - }, - "id": 2777, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "scheduler-{{instance}}", - "metric": "", - "refId": "A", - "step": 4 - }, - { - "expr": "sum(rate(tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "channelfull-{{instance}}-{{type}}", - "metric": "", - "refId": "B", - "step": 4 - }, - { - "expr": "sum(rate(tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type='full'}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "coprocessor-{{instance}}", - "metric": "", - "refId": "C", - "step": 4 - }, - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "stall-{{instance}}", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Server is busy", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 5 - }, - "height": "", - "id": 193, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "maxPerRow": 2, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(tikv_scheduler_contex_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "", - "refId": "A", - "step": 40 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Scheduler pending commands", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 3, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 13 - }, - "id": 2779, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{type}}", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Channel full", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 13 - }, - "id": 2785, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_max\"}) by (db)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{db}} max", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\"}) by (db)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{db}} 99%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile95\"}) by (db)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{db}} 95%", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_average\"}) by (db)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{db}} avg", - "metric": "", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "RocksDB write stall duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "label": null, - "logBase": 10, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "Server Is Busy", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 2801, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 38 - }, - "id": 2768, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (le, type))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "99% gRPC message duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 10, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 3.6 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "0m", - "frequency": "1m", - "handler": 1, - "message": "TiKV gRPC poll thread CPU is high", - "name": "TiKV gRPC poll CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 38 - }, - "id": 2782, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"grpc.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.9 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "gRPC poll CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 7.2 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "", - "frequency": "1m", - "handler": 1, - "message": "TiKV Coprocessor thread CPU is high", - "name": "TiKV Coprocessor CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 45 - }, - "id": 2762, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_normal.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} - normal", - "refId": "A", - "step": 4 - }, - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_high.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} - high", - "refId": "B", - "step": 4 - }, - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_low.*\"}[2m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} - low", - "refId": "C", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.9 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Coprocessor CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 3.6 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "for": "", - "frequency": "1m", - "handler": 1, - "message": "TiKV Storage ReadPool thread CPU is high", - "name": "TiKV Storage ReadPool CPU alert", - "noDataState": "ok", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 45 - }, - "id": 2761, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_norm.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - normal", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - }, - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_high.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - high", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "B", - "step": 4 - }, - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_low.*\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - low", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "C", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.9 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Storage ReadPool CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 5, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 52 - }, - "id": 16, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": null, - "sortDesc": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-99.99%", - "refId": "E" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-99%", - "refId": "A", - "step": 4 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-95%", - "refId": "B", - "step": 4 - }, - { - "expr": " sum(rate(tikv_coprocessor_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (req) / sum(rate(tikv_coprocessor_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (req)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-avg", - "refId": "C", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Coprocessor request duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 5, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 52 - }, - "id": 111, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": null, - "sortDesc": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-99.99%", - "refId": "D" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-99%", - "refId": "A", - "step": 4 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-95%", - "refId": "B", - "step": 4 - }, - { - "expr": " sum(rate(tikv_coprocessor_request_wait_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (req) / sum(rate(tikv_coprocessor_request_wait_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (req)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-avg", - "refId": "C", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Coprocessor wait duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 59 - }, - "id": 52, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.9999, avg(rate(tikv_coprocessor_scan_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, req)) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-99.99%", - "refId": "D" - }, - { - "expr": "histogram_quantile(0.99, avg(rate(tikv_coprocessor_scan_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, req)) ", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{req}}-99%", - "metric": "tikv_coprocessor_scan_keys_bucket", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.95, avg(rate(tikv_coprocessor_scan_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, req)) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-95%", - "metric": "tikv_coprocessor_scan_keys_bucket", - "refId": "B", - "step": 10 - }, - { - "expr": "histogram_quantile(0.90, avg(rate(tikv_coprocessor_scan_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, req)) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{req}}-90%", - "metric": "tikv_coprocessor_scan_keys_bucket", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Coprocessor scan keys", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 5, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 59 - }, - "id": 116, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": null, - "sortDesc": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance,req))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{req}}", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "95% Coprocessor wait duration by store", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 5, - "description": "读请求执行前，都需要拿一个 snapshot", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 66 - }, - "id": 2828, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": null, - "sortDesc": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.999999, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99.9999%", - "refId": "B", - "step": 4 - }, - { - "expr": "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "average", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Get snapshot duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 66 - }, - "id": 82, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"get_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"get_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"get_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"get_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "RocksDB get duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "label": null, - "logBase": 2, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 73 - }, - "id": 125, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"seek_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"seek_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"seek_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"seek_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "RocksDB seek duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "label": null, - "logBase": 2, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "Disk seconds Read Latency.\n- Critical:\n - Recommended performance value is < 10ms as avg value of the Avg Disk sec/Read,Write.\n - Critical value of the Avg Disk sec/Read,Write is > 50ms, should not exceed this value.", - "fill": 1, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 80 - }, - "id": 2821, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(rate(node_disk_read_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])/ rate(node_disk_reads_completed_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) > 0", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}: [{{ device }}]", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Disk Read Latency (ms)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "", - "fill": 1, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 84 - }, - "id": 2822, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(node_disk_read_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m]) + irate(node_disk_written_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m]) > 0", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}: [{{ device }}]", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Disk Throughput", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "Read Too Slow", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 2802, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 7 - }, - "id": 2781, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (le, type, instance))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} {{type}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "99% gRPC message duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 10, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 7 - }, - "id": 109, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le, instance))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}} 99%", - "refId": "A", - "step": 4 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le, instance))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}} 95%", - "refId": "B", - "step": 4 - }, - { - "expr": "rate(tikv_storage_engine_async_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m]) / rate(tikv_storage_engine_async_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} avg", - "refId": "C", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Storage async write duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 14 - }, - "id": 2753, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"prewrite\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 99%", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"prewrite\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 95%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "expr": "rate(tikv_scheduler_latch_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"prewrite\"}[2m]) / rate(tikv_scheduler_latch_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"prewrite\"}[2m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} avg", - "metric": "", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Prewrite latch wait duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 2774, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"commit\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 99%", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"commit\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 95%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "expr": "rate(tikv_scheduler_latch_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"commit\"}[2m]) / rate(tikv_scheduler_latch_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"commit\"}[2m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} avg", - "metric": "", - "refId": "C", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Commit latch wait duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 21 - }, - "id": 2788, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 99%", - "metric": "", - "refId": "A", - "step": 4 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 95%", - "refId": "B", - "step": 4 - }, - { - "expr": "rate(tikv_raftstore_append_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tikv_raftstore_append_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} avg", - "refId": "C", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Append log duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 21 - }, - "id": 2790, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 99%", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Append log duration per server", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 2830, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "B" - }, - { - "expr": "sum(rate(tikv_raftstore_commit_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_commit_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Commit log duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 28 - }, - "id": 2831, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 99%", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Commit log duration per server", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 35 - }, - "id": 31, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 99%", - "metric": "", - "refId": "A", - "step": 4 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 95%", - "refId": "B", - "step": 4 - }, - { - "expr": "rate(tikv_raftstore_apply_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tikv_raftstore_apply_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} avg", - "refId": "C", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Apply log duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 35 - }, - "id": 32, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 99%", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Apply log duration per server", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 42 - }, - "id": 2794, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 99%", - "metric": "", - "refId": "A", - "step": 4 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 95%", - "refId": "B", - "step": 4 - }, - { - "expr": "rate(tikv_raftstore_request_wait_time_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tikv_raftstore_request_wait_time_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} avg", - "refId": "C", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Propose wait duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 42 - }, - "id": 2795, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 99%", - "metric": "", - "refId": "A", - "step": 4 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 95%", - "refId": "B", - "step": 4 - }, - { - "expr": "rate(tikv_raftstore_apply_wait_time_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tikv_raftstore_apply_wait_time_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} avg", - "refId": "C", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Apply wait duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 126, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"write_max\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} max", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"write_percentile99\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 99%", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"write_percentile95\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"write_average\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} avg", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "KV RocksDB write duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "label": null, - "logBase": 2, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 2776, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"write_max\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} max", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"write_percentile99\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 99%", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"write_percentile95\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"write_average\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} avg", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Raft RocksDB write duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "label": null, - "logBase": 2, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 56 - }, - "id": 137, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(tikv_engine_wal_file_synced{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\"}[2m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} sync", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Raft RocksDB WAL sync operations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 135, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxPerRow": 2, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"wal_file_sync_max\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} max", - "refId": "A", - "step": 10 - }, - { - "expr": "tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"wal_file_sync_percentile99\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 99%", - "refId": "B", - "step": 10 - }, - { - "expr": "tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"wal_file_sync_percentile95\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 95%", - "refId": "C", - "step": 10 - }, - { - "expr": "tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"wal_file_sync_average\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} avg", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Raft RocksDB WAL sync duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "label": null, - "logBase": 10, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 63 - }, - "id": 2793, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_max\"}) by (db)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{db}} max", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\"}) by (db)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{db}} 99%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile95\"}) by (db)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{db}} 95%", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_average\"}) by (db)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{db}} avg", - "metric": "", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "RocksDB write stall duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "label": null, - "logBase": 10, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 70 - }, - "id": 2791, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m]) > 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} - {{device}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "IO utilization", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 74 - }, - "id": 2818, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(node_disk_reads_completed_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m]) + rate(node_disk_writes_completed_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}} IOPs", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "IOPs", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "DISK seconds Read/ Write Latency.\n- Critical:\n - Recommended performance value is < 10ms as avg value of the Avg Disk sec/Read,Write.\n - Critical value of the Avg Disk sec/Read,Write is > 50ms, should not exceed this value.", - "fill": 1, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 78 - }, - "id": 2820, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(rate(node_disk_write_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])/ rate(node_disk_writes_completed_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) > 0", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}: [{{ device }}]", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Disk Write Latency (ms)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "description": "", - "fill": 1, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 82 - }, - "id": 2819, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(node_disk_read_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m]) + irate(node_disk_written_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m]) > 0", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}: [{{ device }}]", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Disk Throughput", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "Write Too Slow", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 2806, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 2810, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(tikv_engine_num_files_at_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", level=\"0\"}) by (instance, cf)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}} {{cf}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Level0 SST file number", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 2811, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(tikv_engine_num_immutable_mem_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (instance, cf)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}} {{cf}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Immutable mem-table number", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 15 - }, - "id": 2808, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "tikv_engine_pending_compaction_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{instance}} {{cf}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Pending compaction bytes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 15 - }, - "id": 2812, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_max\", db=\"$db\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} max", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\", db=\"$db\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 99%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile95\", db=\"$db\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} 95%", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_average\", db=\"$db\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} avg", - "metric": "", - "refId": "D", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "RocksDB write stall duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "µs", - "label": null, - "logBase": 10, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": "db", - "title": "Write Stall - $db", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 2803, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 102, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\"}) by(cf)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cf}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Block cache size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 2770, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", app_kubernetes_io_component=\"tikv\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "OOM", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 2804, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 1, - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 1481, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tikv_raftstore_region_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_region_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "D", - "step": 10 - }, - { - "expr": "histogram_quantile(0.999999, sum(rate(tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99.9999%", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Approximate Region size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 2792, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"split_check\"}[2m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Split checker CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "repeat": null, - "title": "Huge Region", - "type": "row" - } - ], - "refresh": "1m", - "schemaVersion": 18, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": { }, - "datasource": null, - "hide": 2, - "includeAll": false, - "label": "K8s-cluster", - "multi": false, - "name": "k8s_cluster", - "options": [ ], - "query": "label_values(tikv_engine_block_cache_size_bytes, k8s_cluster)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { }, - "datasource": null, - "hide": 2, - "includeAll": false, - "label": "tidb_cluster", - "multi": false, - "name": "tidb_cluster", - "options": [ ], - "query": "label_values(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": null, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "db", - "multi": true, - "name": "db", - "options": [], - "query": "label_values(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, db)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": null, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "command", - "multi": true, - "name": "command", - "options": [], - "query": "label_values(tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, type)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": {}, - "datasource": null, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Instance", - "multi": true, - "name": "instance", - "options": [], - "query": "label_values(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "TiKV-Trouble-Shooting", - "uid": "Lg4wiEkZz", - "version": 1 - },"overwrite": false -} \ No newline at end of file diff --git a/rootfs/usr/share/grafana/start-grafana b/rootfs/usr/share/grafana/start-grafana index 57a6b5d..0ac48d1 100755 --- a/rootfs/usr/share/grafana/start-grafana +++ b/rootfs/usr/share/grafana/start-grafana @@ -118,13 +118,9 @@ if [ "on-cluster" != "${REGISTRY_LOCATION}" ]; then echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "REGISTRY-PROXY"))')" > "${DCD}" fi if [ "on-cluster" != "${STORAGE_LOCATION}" ]; then - rm -rf "${DASHBOARD_LOCATION}"/pd.json "${DASHBOARD_LOCATION}"/tikv.json + rm -rf "${DASHBOARD_LOCATION}"/drycc_storage.json # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "STORAGE-MINIO"))')" > "${DCD}" - # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "STORAGE-META-PD"))')" > "${DCD}" - # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "STORAGE-META-TIKV"))')" > "${DCD}" + echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "STORAGE"))')" > "${DCD}" fi for filename in "${DASHBOARD_LOCATION}"/*.json; do From 5c03e2218c3ab2c9e2d97d50322d0cfd26bf9372 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Mon, 12 May 2025 09:27:46 +0800 Subject: [PATCH 15/50] chore(charts): change resources format --- charts/grafana/templates/grafana-deployment.yaml | 12 +++--------- charts/grafana/values.yaml | 9 +++++++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/charts/grafana/templates/grafana-deployment.yaml b/charts/grafana/templates/grafana-deployment.yaml index 4b8446e..846db71 100644 --- a/charts/grafana/templates/grafana-deployment.yaml +++ b/charts/grafana/templates/grafana-deployment.yaml @@ -37,16 +37,10 @@ spec: command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }} args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }} {{- end }} - {{- if or (.Values.limitsCpu) (.Values.limitsMemory)}} + {{- with index .Values "resources" }} resources: - limits: - {{- if (.Values.limitsCpu) }} - cpu: {{.Values.limitsCpu}} - {{- end}} - {{- if (.Values.limitsMemory) }} - memory: {{.Values.limitsMemory}} - {{- end}} - {{- end}} + {{- toYaml . | nindent 10 }} + {{- end }} ports: - containerPort: 3000 name: grafana diff --git a/charts/grafana/values.yaml b/charts/grafana/values.yaml index 4839b68..87497e9 100644 --- a/charts/grafana/values.yaml +++ b/charts/grafana/values.yaml @@ -18,8 +18,13 @@ diagnosticMode: args: - infinity -# limitsCpu: "100m" -# limitsMemory: "50Mi" +resources: {} + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi nodeAffinityPreset: key: "drycc.cc/node" From 3a6368c8c6d2f57a5f8537f41e5221e685c57f45 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 13 May 2025 13:07:10 +0800 Subject: [PATCH 16/50] chore(charts): change prometheus url --- charts/grafana/templates/_helpers.tmpl | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/charts/grafana/templates/_helpers.tmpl b/charts/grafana/templates/_helpers.tmpl index 0fd355a..f6c6931 100644 --- a/charts/grafana/templates/_helpers.tmpl +++ b/charts/grafana/templates/_helpers.tmpl @@ -9,18 +9,8 @@ env: - name: "PROMETHEUS_URL" value: "{{ .Values.prometheusUrl }}" {{- else if .Values.prometheus.enabled }} -- name: "PROMETHEUS_USERNAME" - valueFrom: - secretKeyRef: - name: prometheus-creds - key: username -- name: "PROMETHEUS_PASSWORD" - valueFrom: - secretKeyRef: - name: prometheus-creds - key: password - name: "PROMETHEUS_URL" - value: "http://$(PROMETHEUS_USERNAME):$(PROMETHEUS_PASSWORD)@drycc-prometheus.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:9090" + value: "http://drycc-victoriametrics-vmselect.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:8481/select/0/prometheus" {{- end }} - name: "BIND_PORT" value: "3000" From c462ceb595d3c11180a489acebdcb22e8f1d04e3 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 14 May 2025 14:31:56 +0800 Subject: [PATCH 17/50] fix(grafana): grafana api error --- .../grafana/api/dashboards/drycc_storage.json | 6712 ++++++++--------- 1 file changed, 3159 insertions(+), 3553 deletions(-) diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json b/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json index 3efc1af..ed50bcb 100644 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json @@ -1,3705 +1,3311 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "panel", - "id": "bargauge", - "name": "Bar gauge", - "version": "" - }, - { - "type": "panel", - "id": "gauge", - "name": "Gauge", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "10.3.1" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph (old)", - "version": "" - }, - { - "type": "panel", - "id": "piechart", - "name": "Pie chart", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ + "description": "MinIO Grafana Dashboard - https://min.io/", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 13502, + "graphTooltip": 0, + "id": null, + "links": [ { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "minio" + ], + "type": "dashboards" } - ] - }, - "description": "MinIO Grafana Dashboard - https://min.io/", - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 13502, - "graphTooltip": 0, - "id": null, - "links": [ - { - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "minio" - ], - "type": "dashboards" - } - ], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + ], + "liveNow": false, + "panels": [ + { + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dtdurations", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 1, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "time() - max(minio_node_process_starttime_seconds{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Uptime", + "type": "stat" }, - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" + { + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "percentage", - "steps": [ + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 65, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "sum by (instance) (minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total S3 Ingress", + "type": "stat" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [ { - "color": "green", - "value": null + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" } - ] + ], + "unit": "bytes", + "unitScale": true }, - "unit": "dtdurations", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 0, - "y": 0 - }, - "id": 1, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "time() - max(minio_node_process_starttime_seconds{job=~\"$scrape_jobs\"})", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ + "overrides": [ { - "options": { - "match": "null", - "result": { - "text": "N/A" + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" }, - "type": "special" + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } + ] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 6, + "y": 0 + }, + "id": 50, + "interval": "1m", + "links": [], + "maxDataPoints": 100, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent" ] }, - "unit": "bytes", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 3, - "y": 0 - }, - "id": 65, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (instance) (minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"})", - "format": "table", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Total S3 Ingress", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } + "pluginVersion": "8.2.1", + "targets": [ + { + "exemplar": true, + "expr": "topk(1, sum(minio_cluster_capacity_usable_total_bytes{job=~\"$scrape_jobs\"}) by (instance)) - topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=~\"$scrape_jobs\"}) by (instance))", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "Used", + "refId": "A", + "step": 300 }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "unit": "bytes", - "unitScale": true - }, - "overrides": [ { - "matcher": { - "id": "byName", - "options": "Free" + "exemplar": true, + "expr": "topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=~\"$scrape_jobs\"}) by (instance)) ", + "hide": false, + "interval": "1m", + "legendFormat": "Free", + "refId": "B" + } + ], + "title": "Capacity", + "type": "piechart" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - ] + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true }, - { - "matcher": { - "id": "byName", - "options": "Used" + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Objects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Usage" + ], + "prefix": "All except:", + "readOnly": true } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 6, - "y": 0 - }, - "id": 50, - "interval": "1m", - "links": [], - "maxDataPoints": 100, - "options": { - "displayLabels": [], - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "values": [ - "percent" + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } ] }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "topk(1, sum(minio_cluster_capacity_usable_total_bytes{job=~\"$scrape_jobs\"}) by (instance)) - topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=~\"$scrape_jobs\"}) by (instance))", - "format": "time_series", - "instant": false, - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "Used", - "refId": "A", - "step": 300 + "gridPos": { + "h": 6, + "w": 6, + "x": 10, + "y": 0 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=~\"$scrape_jobs\"}) by (instance)) ", - "hide": false, - "interval": "1m", - "legendFormat": "Free", - "refId": "B" - } - ], - "title": "Capacity", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "pluginVersion": "8.2.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "max(minio_cluster_usage_total_bytes{job=~\"$scrape_jobs\"})", + "interval": "", + "legendFormat": "Usage", + "range": true, + "refId": "A" + } + ], + "title": "Data Usage Growth", + "type": "timeseries" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 16, + "y": 0 + }, + "id": 52, + "links": [], + "options": { + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_objects_size_distribution{job=~\"$scrape_jobs\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{range}}", + "refId": "A", + "step": 300, + "useBackend": false + } + ], + "title": "Object Size Distribution", + "type": "bargauge" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - { - "color": "red", - "value": 80 + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } - ] - }, - "unit": "bytes", - "unitScale": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 61, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "overrides": [ + "pluginVersion": "10.3.1", + "targets": [ { - "matcher": { - "id": "byName", - "options": "Objects" - }, - "properties": [ + "editorMode": "code", + "exemplar": true, + "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{server}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Open FDs ", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" } - ] - }, + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 3 + }, + "id": 64, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Usage" - ], - "prefix": "All except:", - "readOnly": true + "exemplar": true, + "expr": "sum by (instance) (minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total S3 Egress", + "type": "stat" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 } - } - ] + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 3 + }, + "id": 62, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 10, - "y": 0 - }, - "id": 68, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "pluginVersion": "10.3.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "minio_node_go_routine_total{job=~\"$scrape_jobs\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{server}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Goroutines", + "type": "timeseries" }, - "pluginVersion": "8.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(minio_cluster_usage_total_bytes{job=~\"$scrape_jobs\"})", - "interval": "", - "legendFormat": "Usage", - "range": true, - "refId": "A" - } - ], - "title": "Data Usage Growth", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + { + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bool_on_off", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 6 + }, + "id": 94, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_health_status{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "Pool: {{pool}} Set: {{set}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Cluster Health Status", + "transformations": [], + "type": "stat" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-GrYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + { + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 6 + }, + "id": 78, + "links": [], + "maxDataPoints": 100, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "max(minio_cluster_drive_online_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".", + "metric": "process_start_time_seconds", + "range": false, + "refId": "A", + "step": 60 }, - "unitScale": true - }, - "overrides": [] + { + "editorMode": "code", + "exemplar": false, + "expr": "max(minio_cluster_drive_offline_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": ".", + "range": false, + "refId": "B" + } + ], + "title": "Total Online/Offline Drives", + "type": "gauge" }, - "gridPos": { - "h": 6, - "w": 5, - "x": 16, - "y": 0 + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "dark-yellow", + "value": 75000000 + }, + { + "color": "dark-red", + "value": 100000000 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 6 + }, + "id": 66, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "max(minio_cluster_bucket_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of Buckets", + "type": "stat" }, - "id": 52, - "links": [], - "options": { - "displayMode": "lcd", - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 9, + "y": 6 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": true, - "expr": "minio_cluster_objects_size_distribution{job=~\"$scrape_jobs\"}", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{range}}", - "refId": "A", - "step": 300, - "useBackend": false - } - ], - "title": "Object Size Distribution", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "sum by (server) (rate(minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Received [{{server}}]", + "refId": "A" + } + ], + "title": "S3 API Ingress Rate ", + "type": "timeseries" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ + "unit": "binBps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "sum by (server) (rate(minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Sent [{{server}}]", + "refId": "A" + } + ], + "title": "S3 API Egress Rate ", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ { - "color": "green", - "value": null - }, + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 8 + }, + "id": 53, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "max(minio_cluster_nodes_online_total{job=~\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total Online Servers", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ { - "color": "red", - "value": 80 + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" } - ] - }, - "unitScale": true - }, - "overrides": [] + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "dark-yellow", + "value": 75000000 + }, + { + "color": "dark-red", + "value": 100000000 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 9 + }, + "id": 44, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "max(minio_cluster_usage_object_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of Objects", + "type": "stat" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ns", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 10 + }, + "id": 80, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "max(minio_heal_time_last_activity_nano_seconds{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Time Since Last Heal", + "type": "stat" }, - "gridPos": { - "h": 3, - "w": 3, - "x": 21, - "y": 0 + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ns", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 3, + "y": 10 + }, + "id": 81, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "max(minio_usage_last_activity_nano_seconds{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Time Since Last Scan", + "type": "stat" }, - "id": 61, - "links": [], - "maxDataPoints": 100, - "options": { + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 9, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 60, "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 API Request Rate", "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{server}}", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "title": "Open FDs ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" }, - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes", - "unitScale": true + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 7, + "x": 9, + "y": 12 + }, + "hiddenSeries": false, + "id": 88, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 3, - "y": 3 + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_4xx_errors_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 API Request Error Rate (4xx)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } }, - "id": 64, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 86, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, "values": false }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (instance) (minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"})", - "format": "table", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_5xx_errors_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 API Request Error Rate (5xx)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "title": "Total S3 Egress", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "thresholdsStyle": { - "mode": "off" - } + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 99, + "options": { + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_online_drives{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Online Drives", + "range": true, + "refId": "A", + "useBackend": false }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_read_quorum{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Read Quorum", + "range": true, + "refId": "B", + "useBackend": false }, - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 21, - "y": 3 - }, - "id": 62, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "minio_node_go_routine_total{job=~\"$scrape_jobs\"}", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{server}}", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Goroutines", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_write_quorum{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Write Quorum", + "range": true, + "refId": "C", + "useBackend": false }, - "unit": "bool_on_off", - "unitScale": true - }, - "overrides": [] + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_healing_drives{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Healing Drives", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_status{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Status", + "range": true, + "refId": "E", + "useBackend": false + } + ], + "title": "Health Breakdown", + "type": "bargauge" }, - "gridPos": { - "h": 2, - "w": 3, - "x": 0, - "y": 6 + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 76, + "options": { + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "minio_node_process_resident_memory_bytes{job=~\"$scrape_jobs\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{server}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory Usage ", + "type": "bargauge" }, - "id": 94, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 73, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, "values": false }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": true, - "expr": "minio_cluster_health_status{job=~\"$scrape_jobs\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "interval": "", - "legendFormat": "Pool: {{pool}} Set: {{set}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Cluster Health Status", - "transformations": [], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - } - ] + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(minio_node_io_rchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Node RChar [{{server}}]", + "refId": "A" }, - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 6 - }, - "id": 78, - "links": [], - "maxDataPoints": 100, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + { + "exemplar": true, + "expr": "rate(minio_node_io_wchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Node WChar [{{server}}]", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Read, Write I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(minio_cluster_drive_online_total{job=~\"$scrape_jobs\"})", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": ".", - "metric": "process_start_time_seconds", - "range": false, - "refId": "A", - "step": 60 + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(minio_cluster_drive_offline_total{job=~\"$scrape_jobs\"})", - "format": "time_series", - "hide": false, - "instant": true, - "legendFormat": ".", - "range": false, - "refId": "B" + "yaxes": [ + { + "$$hashKey": "object:381", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:382", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false } - ], - "title": "Total Online/Offline Drives", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" + { + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 85 } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 77, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "rate(minio_node_process_cpu_total_seconds{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{server}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "description": "Total number of bytes received and sent on MinIO cluster", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - { - "color": "dark-yellow", - "value": 75000000 + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" }, - { - "color": "dark-red", - "value": 100000000 + "thresholdsStyle": { + "mode": "off" } - ] - }, - "unit": "short", - "unitScale": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 17, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 6, - "y": 6 + "pluginVersion": "10.3.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "rate(minio_inter_node_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Internode Bytes Received [{{server}}]", + "metric": "minio_http_requests_duration_seconds_count", + "range": true, + "refId": "A", + "step": 4 + }, + { + "exemplar": true, + "expr": "rate(minio_inter_node_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Internode Bytes Sent [{{server}}]", + "refId": "B" + } + ], + "title": "Internode Traffic", + "type": "timeseries" }, - "id": 66, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", + { + "aliasColors": { + "available 10.13.1.25:9000": "green", + "used 10.13.1.25:9000": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "", + "fieldConfig": { + "defaults": { + "links": [], + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 33 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, "values": false }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(minio_cluster_bucket_total{job=~\"$scrape_jobs\"})", - "format": "time_series", - "instant": false, - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", + "interval": "", + "legendFormat": "Open FDs [{{server}}]", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false } - ], - "title": "Number of Buckets", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + { + "aliasColors": { + "Offline 10.13.1.25:9000": "dark-red", + "Total 10.13.1.25:9000": "blue" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "description": "Number of online drives per MinIO Server", + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 40 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(minio_node_syscall_read_total{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Read Syscalls [{{server}}]", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 }, - "unit": "binBps", - "unitScale": true + { + "exemplar": true, + "expr": "rate(minio_node_syscall_write_total{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Write Syscalls [{{server}}]", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Syscalls", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 7, - "x": 9, - "y": 6 + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:185", + "decimals": 0, + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:186", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, - "id": 63, - "options": { + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 40 + }, + "hiddenSeries": false, + "id": 95, "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(minio_node_scanner_objects_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Scanned Objects", "tooltip": { - "mode": "single", - "sort": "none" + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false } }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (server) (rate(minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "Data Received [{{server}}]", - "refId": "A" + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 75, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(minio_node_scanner_versions_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Scanned Versions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false } - ], - "title": "S3 API Ingress Rate ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 47 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(minio_node_scanner_directories_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Scanned Directories", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ { - "color": "red", - "value": 80 + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" } - ] - }, - "unit": "binBps", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 6 + ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "dtdurations", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 54 + }, + "id": 89, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "minio_cluster_kms_uptime{job=~\"$scrape_jobs\"}", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "minio_cluster_kms_uptime", + "refId": "A", + "step": 60 + } + ], + "title": "KMS Uptime", + "type": "stat" }, - "id": 70, - "options": { + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 54 + }, + "hiddenSeries": false, + "id": 91, "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (server) (increase(minio_cluster_kms_request_error{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "KMS Request 4xx Error Rate", "tooltip": { - "mode": "single", - "sort": "none" + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (server) (rate(minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "Data Sent [{{server}}]", - "refId": "A" - } - ], - "title": "S3 API Egress Rate ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 0, - "y": 8 - }, - "id": 53, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unit": "bool_on_off", + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 54 + }, + "hiddenSeries": false, + "id": 90, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, "values": false }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "max(minio_cluster_nodes_online_total{job=~\"$scrape_jobs\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Total Online Servers", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "dark-yellow", - "value": 75000000 - }, - { - "color": "dark-red", - "value": 100000000 - } - ] - }, - "unit": "short", - "unitScale": true + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (server) (minio_cluster_kms_online{job=~\"$scrape_jobs\"})", + "interval": "1m", + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "KMS Online(1)/Offline(0)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 6, - "y": 9 - }, - "id": 44, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(minio_cluster_usage_object_total{job=~\"$scrape_jobs\"})", - "format": "time_series", - "instant": false, - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Number of Objects", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "bool_on_off", + "logBase": 1, + "min": "0", + "show": true }, - "unit": "ns", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 0, - "y": 10 - }, - "id": 80, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "max(minio_heal_time_last_activity_nano_seconds{job=~\"$scrape_jobs\"})", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false } - ], - "title": "Time Since Last Heal", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ns", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 3, - "y": 10 }, - "id": 81, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 54 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, "values": false }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "max(minio_usage_last_activity_nano_seconds{job=~\"$scrape_jobs\"})", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Time Since Last Scan", - "type": "stat" - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 9, - "x": 0, - "y": 12 - }, - "hiddenSeries": false, - "id": 60, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (server,api) (increase(minio_s3_requests_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{server,api}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 API Request Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 7, - "x": 9, - "y": 12 - }, - "hiddenSeries": false, - "id": 88, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (server,api) (increase(minio_s3_requests_4xx_errors_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{server,api}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 API Request Error Rate (4xx)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unitScale": true + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(minio_node_scanner_bucket_scans_finished{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Bucket Scans Finished", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 12 - }, - "hiddenSeries": false, - "id": 86, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (server,api) (increase(minio_s3_requests_5xx_errors_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{server,api}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 API Request Error Rate (5xx)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true }, - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 99, - "options": { - "displayMode": "lcd", - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "minio_cluster_health_erasure_set_online_drives{job=~\"$scrape_jobs\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Pool {{pool}} / Set {{set}} - Online Drives", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "minio_cluster_health_erasure_set_read_quorum{job=~\"$scrape_jobs\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Pool {{pool}} / Set {{set}} - Read Quorum", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "minio_cluster_health_erasure_set_write_quorum{job=~\"$scrape_jobs\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Pool {{pool}} / Set {{set}} - Write Quorum", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "minio_cluster_health_erasure_set_healing_drives{job=~\"$scrape_jobs\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Pool {{pool}} / Set {{set}} - Healing Drives", - "range": true, - "refId": "D", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "minio_cluster_health_erasure_set_status{job=~\"$scrape_jobs\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Pool {{pool}} / Set {{set}} - Status", - "range": true, - "refId": "E", - "useBackend": false + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false } - ], - "title": "Health Breakdown", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-GrYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 76, - "options": { - "displayMode": "lcd", - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 58 + }, + "hiddenSeries": false, + "id": 92, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, "values": false }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "minio_node_process_resident_memory_bytes{job=~\"$scrape_jobs\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{server}}", - "range": true, - "refId": "A" - } - ], - "title": "Memory Usage ", - "type": "bargauge" - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 26 - }, - "hiddenSeries": false, - "id": 73, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(minio_node_io_rchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Node RChar [{{server}}]", - "refId": "A" + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (server) (increase(minio_cluster_kms_request_failure{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "KMS Request 5xx Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(minio_node_io_wchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "", - "legendFormat": "Node WChar [{{server}}]", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Read, Write I/O", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:381", - "format": "bytes", - "logBase": 1, - "show": true + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] }, - { - "$$hashKey": "object:382", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green" - }, - { - "color": "orange", - "value": 70 - }, - { - "color": "red", - "value": 85 - } - ] + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true }, - "unit": "s", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 26 - }, - "id": 77, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(minio_node_process_cpu_total_seconds{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{server}}", - "range": true, - "refId": "A" + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" }, - "description": "Total number of bytes received and sent on MinIO cluster", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 17, - "links": [], - "options": { + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 58 + }, + "hiddenSeries": false, + "id": 93, "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (server) (rate(minio_cluster_kms_request_success{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "KMS Request Success [{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "KMS Request Success Rate ", "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(minio_inter_node_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Internode Bytes Received [{{server}}]", - "metric": "minio_http_requests_duration_seconds_count", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(minio_inter_node_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "", - "legendFormat": "Internode Bytes Sent [{{server}}]", - "refId": "B" - } - ], - "title": "Internode Traffic", - "type": "timeseries" - }, - { - "aliasColors": { - "available 10.13.1.25:9000": "green", - "used 10.13.1.25:9000": "blue" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "links": [], - "unit": "bytes", - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 33 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", - "interval": "", - "legendFormat": "Open FDs [{{server}}]", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "File Descriptors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "Offline 10.13.1.25:9000": "dark-red", - "Total 10.13.1.25:9000": "blue" - }, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of online drives per MinIO Server", - "fieldConfig": { - "defaults": { - "links": [], - "unitScale": true + "shared": true, + "sort": 0, + "value_type": "individual" }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 40 - }, - "hiddenSeries": false, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(minio_node_syscall_read_total{job=~\"$scrape_jobs\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Read Syscalls [{{server}}]", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(minio_node_syscall_write_total{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "", - "legendFormat": "Write Syscalls [{{server}}]", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Syscalls", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:185", - "decimals": 0, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:186", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 40 - }, - "hiddenSeries": false, - "id": 95, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(minio_node_scanner_objects_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "1m", - "legendFormat": "[{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scanned Objects", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 47 - }, - "hiddenSeries": false, - "id": 75, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(minio_node_scanner_versions_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "1m", - "legendFormat": "[{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scanned Versions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 47 - }, - "hiddenSeries": false, - "id": 96, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(minio_node_scanner_directories_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "1m", - "legendFormat": "[{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scanned Directories", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green" - } - ] + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true }, - "unit": "dtdurations", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 54 - }, - "id": 89, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "minio_cluster_kms_uptime{job=~\"$scrape_jobs\"}", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "metric": "minio_cluster_kms_uptime", - "refId": "A", - "step": 60 - } - ], - "title": "KMS Uptime", - "type": "stat" - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 54 - }, - "hiddenSeries": false, - "id": 91, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (server) (increase(minio_cluster_kms_request_error{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{server}}", - "refId": "A" + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false } - ], - "thresholds": [], - "timeRegions": [], - "title": "KMS Request 4xx Error Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 61 + }, + "hiddenSeries": false, + "id": 97, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unit": "bool_on_off", - "unitScale": true + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(minio_node_scanner_bucket_scans_started{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Bucket Scans Started", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 54 - }, - "hiddenSeries": false, - "id": 90, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (server) (minio_cluster_kms_online{job=~\"$scrape_jobs\"})", - "interval": "1m", - "legendFormat": "{{server}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "KMS Online(1)/Offline(0)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "bool_on_off", - "logBase": 1, - "min": "0", - "show": true + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false } - ], - "yaxis": { - "align": false } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 54 - }, - "hiddenSeries": false, - "id": 98, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(minio_node_scanner_bucket_scans_finished{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "1m", - "legendFormat": "[{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Bucket Scans Finished", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "minio" + ], + "templating": { + "list": [ + { + "current": {}, + "definition": "label_values(job)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "scrape_jobs", + "options": [], + "query": { + "query": "label_values(job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" } - ], - "yaxis": { - "align": false - } + ] }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 58 - }, - "hiddenSeries": false, - "id": 92, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (server) (increase(minio_cluster_kms_request_failure{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{server}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "KMS Request 5xx Error Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "time": { + "from": "now-6h", + "to": "now" }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 58 - }, - "hiddenSeries": false, - "id": 93, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (server) (rate(minio_cluster_kms_request_success{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "KMS Request Success [{{server}}]", - "refId": "A" - } + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" ], - "thresholds": [], - "timeRegions": [], - "title": "KMS Request Success Rate ", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 61 - }, - "hiddenSeries": false, - "id": 97, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(minio_node_scanner_bucket_scans_started{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "1m", - "legendFormat": "[{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Bucket Scans Started", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "refresh": "", - "schemaVersion": 39, - "tags": [ - "minio" - ], - "templating": { - "list": [ - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(job)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "scrape_jobs", - "options": [], - "query": { - "query": "label_values(job)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] + "timezone": "", + "title": "MinIO Dashboard", + "uid": "TgmJnqnnk", + "version": 54, + "weekStart": "" }, - "timezone": "", - "title": "MinIO Dashboard", - "uid": "TgmJnqnnk", - "version": 54, - "weekStart": "" -} \ No newline at end of file + "overwrite": false +} From e137c32f14104e3d133be1e8a1b3262f329e33b0 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Thu, 15 May 2025 17:15:53 +0800 Subject: [PATCH 18/50] chore(grafana): add victoriametrics --- .../drycc_victoriametrics_cluster.json | 10231 ++++++++++++++++ .../drycc_victoriametrics_vmagent.json | 7466 +++++++++++ rootfs/usr/share/grafana/start-grafana | 4 + 3 files changed, 17701 insertions(+) create mode 100644 rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_cluster.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_vmagent.json diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_cluster.json b/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_cluster.json new file mode 100644 index 0000000..e76ae97 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_cluster.json @@ -0,0 +1,10231 @@ +{ + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "enable": true, + "expr": "sum(ALERTS{job=~\"$job\", instance=~\"$instance\", alertgroup=\"vmcluster\",alertstate=\"firing\",show_at=\"dashboard\"}) by(alertname)", + "hide": false, + "iconColor": "red", + "name": "alerts", + "titleFormat": "{{alertname}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "enable": true, + "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(version))", + "hide": true, + "iconColor": "dark-blue", + "name": "version change", + "textFormat": "{{version}}", + "titleFormat": "Version change" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "enable": true, + "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job)", + "hide": false, + "iconColor": "dark-yellow", + "name": "restarts", + "textFormat": "{{job}} restarted" + } + ] + }, + "description": "Overview for cluster VictoriaMetrics v1.102.0 or higher", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 1, + "links": [ + { + "icon": "doc", + "tags": [], + "targetBlank": true, + "title": "Cluster Wiki", + "type": "link", + "url": "https://docs.victoriametrics.com/cluster-victoriametrics" + }, + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "Found a bug?", + "type": "link", + "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues" + }, + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "New releases", + "type": "link", + "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 137, + "panels": [], + "title": "Stats", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "How many [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) are in storage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 131, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "sum(vm_rows{job=~\"$job_storage\", type!~\"indexdb.*\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total datapoints", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the datapoints ingestion rate, including replication factor.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 124, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(vm_vminsert_metrics_read_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) ", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Ingestion rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of HTTP read requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 130, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_http_requests_total{job=~\"$job\", instance=~\"$instance\", path=~\"/select/.*\"}[$__rate_interval]))", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Read requests", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Total number of available CPUs for all VM components. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 126, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Available CPU", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series) with new data points inserted during the last hour. High value may result in ingestion slowdown.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 34, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(max_over_time(vm_cache_entries{job=~\"$job\", instance=~\"$instance\", type=\"storage/hour_metric_ids\"}[1h]))", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Active series", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Total amount of used disk space", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 4 + }, + "id": 35, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": false, + "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Disk space usage", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Average disk usage per datapoint.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 4 + }, + "id": 112, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\"}) / sum(vm_rows{job=~\"$job_storage\", type!~\"indexdb.*\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Bytes per point", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Total size of available memory for all VM components.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 4 + }, + "id": 128, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Available memory", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "minWidth": 50 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Count" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 149, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Count" + } + ] + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 16, + "x": 8, + "y": 7 + }, + "id": 62, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(min_over_time(up{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)", + "format": "time_series", + "instant": false, + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "Uptime ($job)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "How many [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) are inserted into cluster per second by protocol before the replication. Check vminsert metrics if there are any issues with ingestion.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (type) > 0 ", + "interval": "", + "legendFormat": "{{type}}", + "range": true, + "refId": "A" + } + ], + "title": "Datapoints ingestion rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series) with new data points inserted during the last hour across all storage nodes. High value may result in ingestion slowdown and high memory usage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(vm_cache_entries{job=~\"$job\", instance=~\"$instance\", type=\"storage/hour_metric_ids\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active time series", + "refId": "A" + } + ], + "title": "Active time series ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The less time it takes is better.\n* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(vm_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", quantile=\"0.99\"}) by (path) > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Query duration 0.99 quantile ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 52, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, path) > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{path}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Requests error rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=203&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 104, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, level) > 0", + "format": "time_series", + "hide": false, + "interval": "5m", + "intervalFactor": 1, + "legendFormat": "{{job}} - {{level}}", + "range": true, + "refId": "A" + } + ], + "title": "Logging rate", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 46, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Percentage of used RSS memory (resident).\nThe RSS memory shows the amount of memory recently accessed by the application. It includes anonymous memory and data from recently accessed files (aka page cache).\nThe application's performance will significantly degrade when memory usage is close to 100%.\n\nClick on the line and choose Drilldown to show memory usage per instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 66, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RSS memory % usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%\n\nClick on the line and choose Drilldown to show memory usage per instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=190&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 138, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RSS anonymous memory % usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 160 + }, + "id": 64, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of bytes read/write from the storage layer.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/read .*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 160 + }, + "id": 122, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "read {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Disk writes/reads ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percentage of open file descriptors compared to the limit set in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/max.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 168 + }, + "id": 117, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(process_open_fds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_max_fds{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Open FDs usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of read/write syscalls such as read, pread, write, pwrite.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/read .*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 168 + }, + "id": 204, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(process_io_read_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "read {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_write_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Disk write/read calls ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 176 + }, + "id": 68, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(max_over_time(go_goroutines{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 176 + }, + "id": 119, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(max_over_time(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP connections ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 184 + }, + "id": 70, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(max_over_time(process_num_threads{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Threads ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 184 + }, + "id": 120, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "TCP connections rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/troubleshooting/ ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 192 + }, + "id": 210, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(\n rate(go_gc_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n ) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU spent on GC ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 192 + }, + "id": 213, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, instance, le))) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Go scheduling latency", + "type": "timeseries" + } + ], + "title": "Resource usage ($job)", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 106, + "panels": [ + { + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 211, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "See [Troubleshooting](https://docs.victoriametrics.com/troubleshooting/) docs.", + "mode": "markdown" + }, + "pluginVersion": "11.5.0", + "title": "", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate and total number of new series created over last 24h.\n\nHigh [churn rate](https://docs.victoriametrics.com/faq/#what-is-high-churn-rate) tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected [cardinality](https://docs.victoriametrics.com/keyconcepts/#cardinality) \"explosions\".\n\nThe higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.\n\nTo investigate stats about most expensive series use `api/v1/status/tsdb` handler. More details here https://docs.victoriametrics.com/cluster-victoriametrics/#url-format\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://valyala.medium.com/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "new series over 24h" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisSoftMin", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 104 + }, + "id": 102, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "sum(rate(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "churn rate", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "sum(increase(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[24h]))", + "hide": false, + "interval": "", + "legendFormat": "new series over 24h", + "refId": "B" + } + ], + "title": "Churn rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The percentage of [slow inserts](https://docs.victoriametrics.com/faq/#what-is-a-slow-insert) compared to the total ingestion rate. \\n\\nThe lower the better. \\n\\nIn short, slow insert is a cache miss. There are following reasons for slow inserts to go up: \\n* Ingestion of completely new, not seen before time series;\\n* [Re-routing](https://docs.victoriametrics.com/cluster-victoriametrics/#cluster-availability) of series when one or more vmstorage nodes are unavailable;\\n* Not enough memory to maintain big enough caches for the current workload.\\n\\nIf percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series). \\n\\nSee [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 104 + }, + "id": 108, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(\n rate(vm_slow_row_inserts_total{job=~\"$job_storage\"}[$__rate_interval]) \n / rate(vm_rows_added_to_storage_total{job=~\"$job_storage\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "slow inserts", + "range": true, + "refId": "A" + } + ], + "title": "Slow inserts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Merge assist happens when vmstorage can't keep up with merging parts. This is usually a sign of overload for vmstorage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 112 + }, + "id": 170, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(vm_assisted_merges_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type) > 0", + "format": "time_series", + "interval": "5m", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Assisted merges ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows % of slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.\n\nThe less value is better.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 112 + }, + "id": 107, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_slow_queries_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval]))\n/\nsum(rate(vm_http_requests_total{job=~\"$job_select\", instance=~\"$instance\", path=~\"/select/.*\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "slow queries %", + "range": true, + "refId": "A" + } + ], + "title": "Slow queries % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 120 + }, + "id": 144, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"} \n /\n vm_cache_size_max_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(type)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Cache usage % by type ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows cache miss ratio. Lower is better.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 120 + }, + "id": 58, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n rate(vm_cache_misses_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n rate(vm_cache_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n) by(type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Cache miss ratio ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The value is above 0 when the vmstorage at the given `addr` communicates to the given vminsert node that it cannot accept new data because it is in the read-only mode.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 129 + }, + "id": 142, + "links": [ + { + "targetBlank": true, + "title": "Readonly mode", + "url": "https://docs.victoriametrics.com/cluster-victoriametrics/#readonly-mode" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vm_rpc_vmstorage_is_read_only{job=~\"$job_insert\", instance=~\"$instance\"}) by(instance, addr) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} => {{addr}}", + "range": true, + "refId": "A" + } + ], + "title": "Storage in readonly status for vminsert ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows rate of deduplicated samples during [deduplication](https://docs.victoriametrics.com/#deduplication) or [downsampling](https://docs.victoriametrics.com/#downsampling), according to the configured `-dedup.minScrapeInterval` or `-downsampling.period`.\n\nShould be empty if `-dedup.minScrapeInterval` and `-downsampling.period` isn't set.\n\nDeduplication happens in two places:\n* `type=\"merge\"` - during [background merges](https://docs.victoriametrics.com/#storage) by vmstorages. Deduplication during merges is permanent.\n* `type=\"select\"` - during [read queries](https://docs.victoriametrics.com/keyconcepts/#query-data) by vmselects. Deduplication happens in-flight and has no permanent effect.\n\nThe more samples need to be deduplicated, the higher will be resource usage on vmselects or vmstorages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 129 + }, + "id": 215, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_deduplicated_samples_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) >0", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Deduplication rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows how many samples were ignored or dropped on insertion due to various reasons:\n* timestamp out of retention period or timestamp in future;\n* invalid metric name;\n* exceeding limit for labels length or number;\n* dropped by [relabeling configuration](https://docs.victoriametrics.com/#relabeling).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 137 + }, + "id": 135, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(vm_rows_ignored_total{job=~\"$job\", instance=~\"$instance\"}[1h])) by (reason)", + "hide": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(increase(vm_relabel_metrics_dropped_total{job=~\"$job\", instance=~\"$instance\"}[1h]))", + "hide": false, + "instant": false, + "legendFormat": "relabeling", + "range": true, + "refId": "B" + } + ], + "title": "Samples dropped for last 1h ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of partial results served by `vmselects`.\n\n`vmselect` continues serving queries if at least a single vmstorage nodes is available. It marks responses as `partial` for queries served from the remaining healthy vmstorage nodes.\n\nIf you prefer consistency over availability then run vmselect nodes with `-search.denyPartialResponse` cmd-line flag. In this case vmselect returns an error if at least a single vmstorage node is unavailable.\n\nSee more at [cluster availability](https://docs.victoriametrics.com/cluster-victoriametrics/#cluster-availability).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 137 + }, + "id": 217, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(increase(vm_partial_results_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0", + "format": "time_series", + "instant": false, + "legendFormat": "partial results", + "refId": "A" + } + ], + "title": "Partial query results ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 145 + }, + "id": 183, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "job" + } + ] + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(flag{is_set=\"true\", job=~\"$job\", instance=~\"$instance\"}) by(job, instance, name, value)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Non-default flags", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "instance": { + "aggregations": [ + "uniqueValues" + ], + "operation": "aggregate" + }, + "job": { + "aggregations": [], + "operation": "groupby" + }, + "name": { + "aggregations": [], + "operation": "groupby" + }, + "value": { + "aggregations": [], + "operation": "groupby" + } + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 145 + }, + "id": 214, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "instant": false, + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "Restarts ($job)", + "type": "timeseries" + } + ], + "title": "Troubleshooting", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 48, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows number of pushed and sent rows. \n* `Pushed rows` - rows added to internal inserter buffers before send\n* `Sent rows` - successfully transmitted rows to storage nodes\n\nPlease note, it could be that `Sent > Pushed` because of the replication factor.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1331 + }, + "id": 76, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_rpc_rows_pushed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pushed", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_rpc_rows_sent_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Sent", + "range": true, + "refId": "E" + } + ], + "title": "Rows ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Rate of RPC errors by type:\n* `Connection` - the number of connection errors to vmstorage node\n* `Dial` - the number of dial errors to vmstorage node.\n* `Handshake` - the number of handshake errors to vmstorage node\n* `Rerouted` - errors appeared during rerouting of rows from un-healthy storage node to a healthy one.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1331 + }, + "id": 86, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_rpc_connection_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Connection", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(rate(vm_rpc_dial_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Dial", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(rate(vm_rpc_handshake_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Handshake", + "refId": "E" + } + ], + "title": "RPC errors ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The number of rows rerouted to the vmstorage node from other nodes when they were unhealthy.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1340 + }, + "id": 80, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_rpc_rows_rerouted_to_here_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Rows ($instance) rerouted to ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The number of rows rerouted from the vmstorage node to healthy nodes when the given node was unhealthy.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1340 + }, + "id": 78, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "sum(rate(vm_rpc_rows_rerouted_from_here_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{addr}}", + "refId": "A" + } + ], + "title": "Rows ($instance) rerouted from", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The number of rows or bytes that vminsert internal buffer contains at the moment.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "bytes" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1348 + }, + "id": 82, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(vm_rpc_buf_pending_bytes{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "bytes", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(vm_rpc_rows_pending{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "rows", + "refId": "B" + } + ], + "title": "Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Network usage by internal VictoriaMetrics RPC protocol", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1348 + }, + "id": 74, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) * 8", + "legendFormat": "network usage", + "range": true, + "refId": "A" + } + ], + "title": "RPC network usage ($instance)", + "type": "timeseries" + } + ], + "title": "Interconnection ($job)", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 24, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of datapoints ingested into storage nodes per second. This metric doesn't show all stored datapoints since some of them may be dropped because of wrong timestamps or decode errors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_vminsert_metrics_read_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) ", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "ingestion rate", + "range": true, + "refId": "A" + } + ], + "title": "Ingestion rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the approx time needed to reach 100% of disk capacity for at least one vmstorage node based on the following params:\n* free disk space (after -storage.minFreeDiskSpaceBytes);\n* row ingestion rate;\n* compression.\n\nNote: this panel doesn't account for deduplication process.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=196&${__url_time_range}&${__all_variables}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 113, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "min((vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"} - vm_free_disk_space_limit_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) \n/ \nignoring(path) (\n rate(vm_rows_added_to_storage_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d])\n * scalar(\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n / \n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n )\n))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "min ETA", + "range": true, + "refId": "A" + } + ], + "title": "Storage full ETA ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=$job_storage&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 151, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"}\n)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "min(\n rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "quantile(0.5,\n rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "median", + "range": true, + "refId": "C" + } + ], + "title": "CPU usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Percentage of used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=$job_storage&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 167, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}\n)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(\n max_over_time(process_resident_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "quantile(0.5,\n max_over_time(process_resident_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "median", + "range": true, + "refId": "C" + } + ], + "title": "Memory usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. The `max` is an internal limit and can't be changed. It is always equal to the number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 212, + "links": [ + { + "targetBlank": true, + "title": "Related discussion", + "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/632" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(vm_concurrent_insert_current{job=~\"$job_storage\", \n instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "current", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(vm_concurrent_insert_capacity{job=~\"$job_storage\", instance=~\"$instance\"})", + "hide": false, + "interval": "", + "legendFormat": "max", + "range": true, + "refId": "B" + } + ], + "title": "Concurrent flushes on disk ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of concurrently executed read requests, where:\n* `max` - equal to `-search.maxConcurrentRequest` cmd-line flag;\n* `current` - current number of concurrent select requests executed by storage.\n\nEach concurrent select request requires RAM, CPU and disk IO resources during processing. Too many concurrent requests could result into OOM exceptions.\n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row).\n\nSee more at https://docs.victoriametrics.com/cluster-victoriametrics/#resource-usage-limits", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 133, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(vm_vmselect_concurrent_requests_current{job=~\"$job_storage\", \n instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "current", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(vm_vmselect_concurrent_requests_capacity{job=~\"$job_storage\", instance=~\"$instance\"})", + "hide": false, + "interval": "", + "legendFormat": "max", + "range": true, + "refId": "B" + } + ], + "title": "Concurrent selects ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The max number of on-going merges across storage nodes.\n The drastic change in number of merges could be a sign of on-going deduplication/downsampling activity.\n It is expected to have high numbers for `storage/small` metric.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 54, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(max_over_time(vm_active_merges{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active merges ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The number of rows merged per second by storage nodes. Merge speed depends on available CPU and disk IO bandwidth.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 55, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_rows_merged_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Merge speed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=200&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "min(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "quantile(0.5,\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "median", + "range": true, + "refId": "C" + } + ], + "title": "Disk space usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The max number of data parts of LSM tree across all storage nodes in the last [partition](https://docs.victoriametrics.com/#storage).\nIncrease in number of parts (the hard limit is 512) is an evidence of slow merge performance - check the resource utilization.\n* `indexdb` - inverted index\n* `storage/small` - recently added parts of data ingested into storage (hot data)\n* `storage/big` - small parts gradually merged into bigger parts (cold data)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(vm_parts{job=~\"$job_storage\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(type)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(vm_last_partition_parts{job=~\"$job_storage\", instance=~\"$instance\"}) by(type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "LSM parts max by type ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percentage of used disk space by type: datapoints or indexdb. Normally, indexdb takes much less space comparing to datapoints. But with high [churn rate](https://docs.victoriametrics.com/faq/#what-is-high-churn-rate) the size of the indexdb could grow significantly.\n\nThe sum of the % can be > 100% since panel shows max % per-job and per-instance. It means different instance can have different ratio between datapoints and indexdb size.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=201&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 202, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(job, instance)\n / \n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "indexdb", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"}) by(job, instance)\n / \n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "datapoints", + "range": true, + "refId": "B" + } + ], + "title": "Disk space usage % by type ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `3*`, since VictoriaMetrics pushes pending data to persistent storage every two seconds. The index datapoints value in general is much lower.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "pending index entries" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "decimals", + "value": 3 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(max_over_time(vm_pending_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"storage\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "pending datapoints", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(max_over_time(vm_pending_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "pending index entries", + "range": true, + "refId": "B" + } + ], + "title": "Pending datapoints ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows network usage by vmstorage services.\n* Writes show traffic sent to clients.\n* Reads show traffic received from clients.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 88 + }, + "id": 206, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(name) * 8 > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "read from {{name}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(name) * 8 > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "write to {{name}}", + "range": true, + "refId": "B" + } + ], + "title": "Network usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Value > 0 means vmstorage is in readonly mode.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 88 + }, + "id": 141, + "links": [ + { + "targetBlank": true, + "title": "Readonly mode", + "url": "https://docs.victoriametrics.com/cluster-victoriametrics/#readonly-mode" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "vm_storage_is_read_only{job=~\"$job_storage\", instance=~\"$instance\"} > 0", + "interval": "", + "legendFormat": "{{ instance }}", + "range": true, + "refId": "A" + } + ], + "title": "Readonly mode", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the max number of existing [snapshots](https://docs.victoriametrics.com/#how-to-work-with-snapshots) across vmstorages.\n\nRecently made snapshots do not occupy disk space. But with time, as snapshots become older, they start to occupy more and more disk space.\n\nIt is recommended deleting old snapshots when they are no longer needed to free up disk space.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 96 + }, + "id": 216, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(vm_snapshots{job=~\"$job_storage\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "snapshots", + "range": true, + "refId": "A" + } + ], + "title": "Number of snapshots", + "type": "timeseries" + } + ], + "title": "vmstorage ($instance)", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 42, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Request rate accepted by vmselect nodes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1344 + }, + "id": 92, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_http_requests_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Requests rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the max number of concurrent selects across instances.\n* `max` limit can be configured via `search.maxConcurrentRequests` flag\n* `current` shows the current number of goroutines busy with processing requests\n\nWhen `current` hits `max` constantly, it means one or more vmselect nodes are overloaded with number of requests. If you observe that CPU for vmselects is saturated, consider adding more vmselect replicas or increase CPU resources. If CPU panel shows a plenty of free resources - try increasing `search.maxConcurrentRequests`.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1344 + }, + "id": 95, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(max_over_time(vm_concurrent_select_current{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) ", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "current", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "min(vm_concurrent_select_capacity{job=~\"$job_select\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "B" + } + ], + "title": "Concurrent selects ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=$job_select&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1352 + }, + "id": 163, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_select\", instance=~\"$instance\"}\n)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(\n rate(process_cpu_seconds_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_select\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "quantile(0.5,\n rate(process_cpu_seconds_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_select\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "median", + "range": true, + "refId": "C" + } + ], + "title": "CPU usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=$job_select&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1352 + }, + "id": 165, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}\n)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(\n max_over_time(process_resident_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "quantile(0.5,\n max_over_time(process_resident_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "median", + "range": true, + "refId": "C" + } + ], + "title": "Memory usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "99th percentile of the number of time series read per query.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1360 + }, + "id": 178, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "series", + "range": true, + "refId": "A" + } + ], + "title": "Series read per query ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "99th percentile of number of [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) read per queried time series.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1360 + }, + "id": 180, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "datapoints", + "range": true, + "refId": "A" + } + ], + "title": "Datapoints read per series ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "99th percentile of number of [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) read per query.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1368 + }, + "id": 179, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "datapoints", + "range": true, + "refId": "A" + } + ], + "title": "Datapoints read per query ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "99th percentile of number of [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) scanner per query.\n\nThis number can exceed number of DatapointsReadPerQuery if `step` query arg passed to [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) is smaller than the lookbehind window set in square brackets of [rollup function](https://docs.victoriametrics.com/metricsql/#rollup-functions). For example, if `increase(some_metric[1h])` is executed with the `step=5m`, then the same [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on a hour time range are scanned `1h/5m=12` times. See [this article](https://valyala.medium.com/how-to-optimize-promql-and-metricsql-queries-85a1b75bf986) for details.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1368 + }, + "id": 181, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "datapoints", + "range": true, + "refId": "A" + } + ], + "title": "Datapoints scanned per query ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows network usage between vmselects and clients, such as vmalert, Grafana, vmui, etc.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1376 + }, + "id": 93, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "read from client", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "write to client", + "range": true, + "refId": "B" + } + ], + "title": "Network usage: clients ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows network usage between vmselects and vmstorages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1376 + }, + "id": 207, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcpdialer_read_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "read from vmstorage", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "write to vmstorage", + "range": true, + "refId": "B" + } + ], + "title": "Network usage: vmstorage ($instance)", + "type": "timeseries" + } + ], + "title": "vmselect ($instance)", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 40, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1345 + }, + "id": 97, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_http_requests_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Requests rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the max number of ongoing insertions.\n* `max` - equal to number of CPU * 2 by default. May be configured with `maxConcurrentInserts` flag;\n* `current` - current number of goroutines busy with processing requests.\n\n`-maxConcurrentInserts` limits the number of insert requests which may be actively processed at any given point in time. All the other insert requests are queued for up to `-insert.maxQueueDuration` in the hope they will get a chance to be processed. This queue is used mostly for absorbing spikes for incoming insert request rate.\n\nWhen `current` hits `max` constantly, it means vminsert node is overloaded and requires more CPU or higher limits.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1345 + }, + "id": 99, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(max_over_time(vm_concurrent_insert_current{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "current", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "min(vm_concurrent_insert_capacity{job=~\"$job_insert\", instance=~\"$instance\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "B" + } + ], + "title": "Concurrent inserts ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=$job_insert&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1353 + }, + "id": 185, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_insert\", instance=~\"$instance\"}\n)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(\n rate(process_cpu_seconds_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_insert\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "quantile(0.5,\n rate(process_cpu_seconds_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_insert\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "median", + "range": true, + "refId": "C" + } + ], + "title": "CPU usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=$job_insert&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1353 + }, + "id": 187, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}\n)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(\n max_over_time(process_resident_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "quantile(0.5,\n max_over_time(process_resident_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "median", + "range": true, + "refId": "C" + } + ], + "title": "Memory usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the saturation level of connection between vminsert and vmstorage components. \n\nIf the threshold of 0.9sec is reached, then the connection is saturated by more than 90% and vminsert won't be able to keep up. This usually means that either vminsert or vmstorage nodes are struggling with the load. Verify CPU/mem saturation of both components and network saturation between them.\nIf vminsert resources are saturated - consider adding more resources or scale vminserts horizontally.\n\nIf vminsert resources and network are fine, check vmstorage metrics for anomalies.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1361 + }, + "id": 139, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(rate(vm_rpc_send_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Storage connection saturation ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows when vmstorage node is unreachable for vminsert.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1361 + }, + "id": 114, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "vm_rpc_vmstorage_is_reachable{job=~\"$job\", instance=~\"$instance\"} != 1", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} => {{addr}}", + "range": true, + "refId": "A" + } + ], + "title": "Storage reachability ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows network usage between vminserts and clients, such as vmagent, Prometheus, or any other client pushing metrics to vminsert.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1369 + }, + "id": 208, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "read from client", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "write to client", + "range": true, + "refId": "B" + } + ], + "title": "Network usage: clients ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows network usage between vminserts and vmstorages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1369 + }, + "id": 209, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcpdialer_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "read from vmstorage", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "write to vmstorage", + "range": true, + "refId": "B" + } + ], + "title": "Network usage: vmstorage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1377 + }, + "id": 88, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(increase(vm_rows_per_insert_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + } + ], + "title": "Rows per insert ($instance)", + "type": "timeseries" + } + ], + "title": "vminsert ($instance)", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 194, + "panels": [ + { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 1394 + }, + "id": 198, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.", + "mode": "markdown" + }, + "pluginVersion": "9.2.7", + "title": "", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1396 + }, + "id": 189, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "RSS memory usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "RSS share for memory allocated by the process itself. This share cannot be freed by the OS, so it must be taken into account by OOM killer.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1396 + }, + "id": 190, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "RSS anonymous memory usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1404 + }, + "id": 192, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "CPU usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the approx time needed to reach 100% of disk capacity based on the following params:\n* free disk space (after -storage.minFreeDiskSpaceBytes);\n* row ingestion rate;\n* compression.\n\nNote: this panel doesn't account for deduplication process.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1404 + }, + "id": 196, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "(vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) \n/ \nignoring(path) (\n rate(vm_rows_added_to_storage_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d])\n * scalar(\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n / \n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n )\n)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Storage full ETA ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1411 + }, + "id": 200, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Disk space usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1411 + }, + "id": 201, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(job, instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{job}}:{{instance}} (indexdb)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"}) by(job, instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{job}}:{{instance}} (datapoints)", + "range": true, + "refId": "B" + } + ], + "title": "Disk space usage by type ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1419 + }, + "id": 203, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, instance, level, location) > 0", + "format": "time_series", + "hide": false, + "interval": "5m", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Logging rate", + "type": "timeseries" + } + ], + "title": "Drilldown", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "VictoriaMetrics - cluster", + "value": "PAF93674D0B4E9963" + }, + "includeAll": false, + "name": "ds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)", + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)", + "refId": "VictoriaMetrics-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vminsert.*\"}, job)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "job_insert", + "options": [], + "query": { + "query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vminsert.*\"}, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmselect.*\"}, job)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "job_select", + "options": [], + "query": { + "query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmselect.*\"}, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmstorage.*\"}, job)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "job_storage", + "options": [], + "query": { + "query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmstorage.*\"}, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{job=~\"$job\"}, instance)", + "includeAll": true, + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(vm_app_version{job=~\"$job\"}, instance)", + "refId": "VictoriaMetrics-instance-Variable-Query" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "filters": [], + "name": "adhoc", + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "VictoriaMetrics - cluster", + "uid": "oS7Bi_0Wz", + "version": 1, + "weekStart": "", + "gnetId": 11176 + } +} \ No newline at end of file diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_vmagent.json b/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_vmagent.json new file mode 100644 index 0000000..3c473aa --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_vmagent.json @@ -0,0 +1,7466 @@ +{ + "dashboard": { + "__inputs": [], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.4.2" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "enable": true, + "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(short_version))", + "hide": true, + "iconColor": "dark-blue", + "name": "version", + "textFormat": "{{short_version}}", + "titleFormat": "Version change" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "enable": true, + "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job, instance)", + "iconColor": "dark-yellow", + "name": "restarts", + "textFormat": "{{job}}:{{instance}} restarted" + } + ] + }, + "description": "Overview for VictoriaMetrics vmagent v1.102.0 or higher", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "icon": "doc", + "tags": [], + "targetBlank": true, + "title": "vmagent wiki", + "tooltip": "", + "type": "link", + "url": "https://docs.victoriametrics.com/vmagent/" + }, + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "Found a bug?", + "type": "link", + "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues" + }, + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "New releases", + "type": "link", + "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 105, + "panels": [], + "title": "Stats", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) scraped from configured targets.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 103, + "links": [ + { + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_scraped_samples_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Samples scraped/s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of ingested [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 102, + "links": [ + { + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmagent_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Samples ingested/s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of targets scraped per second.", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 134, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_scrapes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) ", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Targets scraped/s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows total number of all configured scrape targets in state `up` or `down`.\n\nSee `http://vmagent-host:8429/targets` to get list of all targets. \n", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "down" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 72, + "links": [ + { + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"up\"})", + "interval": "", + "legendFormat": "up", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"down\"})", + "hide": false, + "interval": "", + "legendFormat": "down", + "range": true, + "refId": "B" + } + ], + "title": "Scrape targets", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows number of generated error messages in logs over last 30m. Non-zero value may be a sign of connectivity or missconfiguration errors.", + "fieldConfig": { + "defaults": { + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 16, + "links": [ + { + "targetBlank": true, + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(increase(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[30m]))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Log errors (30m)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Persistent queue size shows size of pending samples in bytes which hasn't been flushed to remote storage yet. \nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 10485760 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 56, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(vm_persistentqueue_bytes_pending{job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Persistent queue size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "minWidth": 50 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Count" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 101, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(up{job=~\"$job\", instance=~\"$instance\"}) by (job)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "__auto", + "refId": "A" + } + ], + "title": "Uptime", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 24, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "refId": "A" + } + ], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows in/out [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) rate including push and pull models. \n\nThe out-rate could be different to in-rate because of replication or additional timeseries added by vmagent for every scraped target.\n\nClick on the line and choose Drilldown to show CPU usage per instance\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "title": "Drilldown", + "url": "/d/G7Z9GzMGz?viewPanel=123&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/out .*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate({__name__=~\"vmagent_rows_inserted_total|vm_promscrape_scraped_samples_sum\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "hide": false, + "interval": "", + "legendFormat": "in {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmagent_remotewrite_block_size_rows_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "interval": "", + "legendFormat": "out {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Samples rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the persistent queue size of pending samples in bytes >2MB which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\nClick on the line and choose Drilldown to show the persistent queue size per instance.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/G7Z9GzMGz?viewPanel=125&var-url=${__field.labels.url}&var-ds=$ds&var-instance=$instance&var-job=$job&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 17, + "links": [ + { + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vmagent_remotewrite_pending_data_bytes{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}) by (job, url) > 2e6", + "interval": "", + "legendFormat": "{{job}} => {{url}}", + "range": true, + "refId": "A" + } + ], + "title": "Persistent queue size ($instance) to ($url)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 107, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, level) > 0", + "format": "time_series", + "hide": false, + "interval": "5m", + "intervalFactor": 1, + "legendFormat": "{{job}} - {{level}}", + "range": true, + "refId": "A" + } + ], + "title": "Logging rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows rate of dropped [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) from persistent queue. vmagent drops samples from queue if in-memory and on-disk queues are full and it is unable to flush them to remote storage.\nThe max size of on-disk queue is configured by `-remoteWrite.maxDiskUsagePerURL` flag.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 49, + "links": [ + { + "targetBlank": true, + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_persistentqueue_bytes_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0", + "interval": "", + "legendFormat": "{{ path }}", + "range": true, + "refId": "A" + } + ], + "title": "Persistent queue dropped rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of requests served by vmagent HTTP server.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmagent_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, path) > 0", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Requests rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Errors rate shows rate for multiple metrics that track possible errors in vmagent, such as network or parsing errors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 69, + "links": [ + { + "targetBlank": true, + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmagent_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, protocol) > 0", + "interval": "", + "legendFormat": "requests:{{protocol}} ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_protoparser_read_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", + "interval": "", + "legendFormat": "parse: {{type}} ({{job}})", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_ingestserver_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", + "interval": "", + "legendFormat": "ingest: {{type}} ({{job}})", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_protoparser_unmarshal_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", + "interval": "", + "legendFormat": "unmarshal: {{type}} ({{job}})", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_dial_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "interval": "", + "legendFormat": "scrape dial ({{job}})", + "range": true, + "refId": "E" + } + ], + "title": "Errors rate ($instance)", + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 45, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Click on the line and choose Drilldown to show CPU usage per instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/G7Z9GzMGz?viewPanel=119&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 109, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Percentage of used RSS memory (resident).\nThe RSS memory shows the amount of memory recently accessed by the application. It includes anonymous memory and data from recently accessed files (aka page cache).\nThe application's performance will significantly degrade when memory usage is close to 100%.\n\nClick on the line and choose Drilldown to show memory usage per instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/G7Z9GzMGz?viewPanel=117&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 111, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RSS memory % usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of bytes read/write from the storage layer when vmagent has to buffer data on disk or read already buffered data.\n\nClick on the line and choose Drilldown to show CPU usage per instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/G7Z9GzMGz?viewPanel=121&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "read" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 81, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "read {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Disk writes/reads ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Network usage shows the bytes rate for data accepted by vmagent and pushed via remotewrite protocol.\nDiscrepancies are possible because of different protocols used for ingesting, scraping and writing data.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) * 8 \n+ sum(rate(vm_promscrape_conn_bytes_read_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) * 8", + "interval": "", + "legendFormat": "in {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmagent_remotewrite_conn_bytes_written_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) * 8", + "interval": "", + "legendFormat": "out {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Network usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Panel shows the percentage of open file descriptors in the OS per instance.\nReaching the limit of open files (100%) can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 5, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 83, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(\n max_over_time(process_open_fds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_max_fds{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Open FDs usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 39, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(max_over_time(go_goroutines{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/troubleshooting/ ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 135, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(\n rate(go_gc_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n ) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU spent on GC ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 149, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Go scheduling latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 41, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(max_over_time(process_num_threads{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Threads ($instance)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "refId": "A" + } + ], + "title": "Resource usage", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 94, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows top 10 jobs by the number of new series registered by vmagent over the 5min range. These jobs generate the most of the churn rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 92, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk_max(10, sum(sum_over_time(scrape_series_added[5m])) by (job)) > 0", + "interval": "", + "legendFormat": "{{ job }}", + "range": true, + "refId": "A" + } + ], + "title": "Top 10 jobs by unique samples", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows top 10 instances by the number of new series registered by vmagent over the 5min range. These instances generate the most of the churn rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 95, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk(10, sum(sum_over_time(scrape_series_added[5m])) by (instance)) > 0", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Top 10 instances by unique samples", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows write saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 98, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(rate(vm_persistentqueue_write_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Persistent queue write saturation ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows read saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with reading data from the disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 99, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(rate(vm_persistentqueue_read_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Persistent queue read saturation ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of dropped data blocks in cases when remote storage replies with `400 Bad Request` and `409 Conflict` HTTP responses.\n\nSee https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 79, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vmagent_remotewrite_packets_dropped_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) by(job, url) > 0", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Data blocks dropped ($instance) to ($url)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of dropped [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) due to relabeling. \nMetric tracks drops for `-remoteWrite.relabelConfig` configuration only.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 18, + "links": [ + { + "targetBlank": true, + "title": "Relabeling", + "url": "https://docs.victoriametrics.com/vmagent/#relabeling" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmagent_remotewrite_relabel_metrics_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, url) > 0", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Rows dropped by relabeling ($instance) to ($url)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of parsed datapoints from write or scrape requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 127, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_protoparser_rows_read_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", + "interval": "", + "legendFormat": "{{ type }} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Datapoints rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Tracks the rate of dropped invalid rows because of errors while unmarshaling write requests. The exact errors messages will be printed in logs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 50, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_rows_invalid_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", + "interval": "", + "legendFormat": "{{type}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Invalid datapoints rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 129, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "job" + } + ] + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(flag{is_set=\"true\", job=~\"$job\", instance=~\"$instance\"}) by(job, instance, name, value)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Non-default flags", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "instance": { + "aggregations": [] + }, + "job": { + "aggregations": [], + "operation": "groupby" + }, + "name": { + "aggregations": [], + "operation": "groupby" + }, + "value": { + "aggregations": [], + "operation": "groupby" + } + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 150, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)", + "format": "time_series", + "instant": false, + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "Restarts ($job)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Troubleshooting", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 28, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 48, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"up\"}) by(job, type) > 0", + "format": "time_series", + "interval": "", + "legendFormat": "{{type}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Scrape targets UP(By Type)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 76, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"down\"}) by(job, type) > 0", + "format": "time_series", + "interval": "", + "legendFormat": "{{type}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Scrape targets DOWN(By Type)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 60 + }, + "id": 132, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vm_promscrape_scrape_pool_targets{job=~\"$job\", instance=~\"$instance\", status=\"up\"}) by(job, scrape_job) > 0", + "format": "time_series", + "interval": "", + "legendFormat": "{{job}}: {{scrape_job}}", + "range": true, + "refId": "A" + } + ], + "title": "Scrape targets UP(By Job)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 60 + }, + "id": 133, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vm_promscrape_scrape_pool_targets{job=~\"$job\", instance=~\"$instance\", status=\"down\"}) by(job, scrape_job) > 0", + "format": "time_series", + "interval": "", + "legendFormat": "{{job}}: {{scrape_job}}", + "range": true, + "refId": "A" + } + ], + "title": "Scrape targets DOWN(By Job)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of scrapes per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_promscrape_scrapes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Scrape rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of datapoints scraped per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 126, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_promscrape_scraped_samples_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Scraped datapoints rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 46, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_response_size_bytes_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)", + "format": "time_series", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Scrape response size 0.99 quantile ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 75 + }, + "id": 148, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)", + "format": "time_series", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Scrape duration 0.99 quantile ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_scrapes_failed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "interval": "", + "legendFormat": "scrapes failed ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_scrapes_timed_out_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "interval": "", + "legendFormat": "timeouts ({{job}})", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_scrapes_gunzip_failed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "interval": "", + "legendFormat": "gunzip fails ({{job}})", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_dial_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "interval": "", + "legendFormat": "dial fails ({{job}})", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_max_scrape_size_exceeded_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "hide": false, + "interval": "", + "legendFormat": "max scrape size exceeded ({{job}})", + "range": true, + "refId": "E" + } + ], + "title": "Scrape fails ($instance)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "refId": "A" + } + ], + "title": "Scraping", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 71, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of write requests served by ingestserver (UDP, TCP connections) and HTTP server.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 73, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_ingestserver_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type, net) > 0", + "interval": "", + "legendFormat": "{{net}}: {{ type }} ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vmagent_http_requests_total{job=~\"$job\", instance=~\"$instance\", protocol!=\"\"}[$__rate_interval])) by(job, protocol) > 0", + "interval": "", + "legendFormat": "{{ protocol }}: http ({{job}})", + "range": true, + "refId": "B" + } + ], + "title": "Requests rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of rows ingested in vmagent via push protocols.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 131, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vmagent_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", + "interval": "", + "legendFormat": "{{ type }} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Rows rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows how many concurrent inserts are taking place.\n\nIf the number of concurrent inserts hitting the `limit` or is close to the `limit` constantly - it might be a sign of a resource shortage.\n\n If vmagent's CPU usage and remote write connection saturation are at normal level, it might be that `-maxConcurrentInserts` cmd-line flag need to be increased.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 130, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max_over_time(vm_concurrent_insert_current{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(vm_concurrent_insert_capacity{job=~\"$job\", instance=~\"$instance\"}) by(job)", + "interval": "", + "legendFormat": "limit ({{job}})", + "range": true, + "refId": "B" + } + ], + "title": "Concurrent inserts ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of write errors in ingestserver (UDP, TCP connections) and HTTP server.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 77, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "sum(rate(vm_ingestserver_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(type, net) > 0", + "interval": "", + "legendFormat": "{{ type }} ({{net}})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "sum(rate(vmagent_http_request_errors_total{job=~\"$job\", instance=~\"$instance\", protocol!=\"\"}[$__rate_interval])) by(protocol) > 0", + "interval": "", + "legendFormat": "{{ protocol }} (http)", + "refId": "B" + } + ], + "title": "Error rate ($instance)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "refId": "A" + } + ], + "title": "Ingestion", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 136, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of matched samples by the aggregation rule. \n\nThe more samples is matched, the more work this aggregation rule does. The matching rule is specified via `match` param.\n\nSee more details in [stream aggregation config](https://docs.victoriametrics.com/stream-aggregation/#stream-aggregation-config). ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 146, + "options": { + "legend": { + "calcs": [ + "min", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_streamaggr_matched_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Matched samples ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The rate of ignored samples during aggregation. \nStream aggregation will drop samples with NaN values, or samples with too old timestamps. See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 143, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_streamaggr_ignored_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0) without (instance, pod)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Ignored samples ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of produced samples by the aggregation rule. \n\nNumber of produced samples depend on params like `by`, `without`, `interval`, etc.\n\nSee more details in [stream aggregation config](https://docs.victoriametrics.com/stream-aggregation/#stream-aggregation-config). ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 147, + "options": { + "legend": { + "calcs": [ + "min", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_streamaggr_output_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Produced samples ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows events when deduplication or aggregation couldn't be finished in the configured interval. Such events may result into bad accuracy of produced data.\n\nPossible solutions:\n* increase interval; \n* use match filter matching smaller number of series;\n* reduce samples ingestion rate to stream aggregation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": -5, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 139, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0", + "instant": false, + "legendFormat": "aggregation: {{url}} ({{job}}): {{path}}:{{position}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0", + "hide": false, + "instant": false, + "legendFormat": "deduplication: {{url}} ({{job}}): {{path}}:{{position}}", + "range": true, + "refId": "B" + } + ], + "title": "Flush timeouts ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the max lag between samples timestamps within one batch passed to the aggregation input.\n\nLower is better.\n\nToo high lag or lag exceeding the interval might be a sign that data was delayed before aggregation or resource insufficiency on aggregator. Samples with high lag may affect accuracy of aggregation.\n\nSee https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 142, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Samples lag 0.99 quantile ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The 99th percentile of avg flush duration for the aggregated data. \n\nSmaller is better.\n\nAggregation can produce incorrect results ff flush duration exceeds configured deduplication interval. See \"Flush Timeouts\" panel.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 137, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, rate(vm_streamaggr_dedup_flush_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Dedup flush duration 0.99 quantile ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the size of Label Compressor in number of entries.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/bytes.*/" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 141, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(vm_streamaggr_labels_compressor_items_count{job=~\"$job\",instance=~\"$instance\"}) by(job, instance)", + "hide": false, + "instant": false, + "legendFormat": "items: {{instance}} ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(vm_streamaggr_labels_compressor_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)", + "hide": false, + "instant": false, + "legendFormat": "bytes: {{instance}} ({{job}})", + "range": true, + "refId": "B" + } + ], + "title": "Labels compressor ($instance)", + "type": "timeseries" + } + ], + "title": "Streaming aggregation", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 58, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of requests to configured remote write endpoints by url and status code.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 60, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vmagent_remotewrite_requests_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) by(job, url, status_code) > 0", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Requests rate ($instance) to ($url)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the global rate for number of written bytes via remote write connections.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 66, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vmagent_remotewrite_conn_bytes_written_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Bytes write rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows requests retry rate by url. Number of retries is unlimited but protected with delays up to 1m between attempts.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 61, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vmagent_remotewrite_retries_count_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) by(url) > 0", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Retry rate ($instance) to ($url)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows current number of established connections to remote write endpoints.\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 65, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(max_over_time(vmagent_remotewrite_conns{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Connections ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the current limit usage of unique series over an hourly period. Vmagent will start to drop series once the limit is reached.\n\nPlease note, panel will be blank if `remoteWrite.maxHourlySeries` is not set.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 88, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n vmagent_hourly_series_limit_current_series{job=~\"$job\", instance=~\"$instance\"} \n / \n vmagent_hourly_series_limit_max_series{job=~\"$job\", instance=~\"$instance\"}\n ) by(job) * 100", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Hourly series limit", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows saturation of every connection to remote storage. If the threshold of 90% is reached, then the connection is saturated (busy or slow) by more than 90%, so vmagent won't be able to keep up and can start buffering data. \n\nThis usually means that `-remoteWrite.queues` command-line flag must be increased in order to increase the number of connections per each remote storage.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 84, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n rate(vmagent_remotewrite_send_duration_seconds_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])\n /\n vmagent_remotewrite_queues{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}\n) by(job, url)", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Remote write connection saturation ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the current limit usage of unique series over a daily period. Vmagent will start to drop series once the limit is reached.\n\nPlease note, panel will be blank if `remoteWrite.maxDailySeries` is not set.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 90, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n vmagent_daily_series_limit_current_series{job=~\"$job\",instance=~\"$instance\"} \n / \n vmagent_daily_series_limit_max_series{job=~\"$job\",instance=~\"$instance\"}\n) by(job)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Daily series limit", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "refId": "A" + } + ], + "title": "Remote write", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 113, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 115, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.", + "mode": "markdown" + }, + "pluginVersion": "10.4.2", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 119, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "CPU usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 117, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "RSS memory usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the persistent queue size of pending samples in bytes which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 125, + "links": [ + { + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vmagent_remotewrite_pending_data_bytes{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}) by (instance, url)", + "interval": "", + "legendFormat": "{{instance}} => {{url}}", + "range": true, + "refId": "A" + } + ], + "title": "Persistent queue size ($instance) to ($url)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows in/out samples rate including push and pull models. \n\nThe out-rate could be different to in-rate because of replication or additional timeseries added by vmagent for every scraped target.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/out .*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 123, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_scraped_samples_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)\n+ sum(rate(vmagent_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", + "hide": false, + "interval": "", + "legendFormat": "in {{instance}} {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmagent_remotewrite_block_size_rows_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", + "interval": "", + "legendFormat": "out {{instance}} {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Samples rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of bytes read/write from the storage layer when vmagent has to buffer data on disk or read already buffered data.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "read" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 121, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "read {{instance}} {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job,instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{instance}} {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Disk writes/reads ($instance)", + "type": "timeseries" + } + ], + "title": "Drilldown", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "vmagent", + "victoriametrics" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "VictoriaMetrics - cluster", + "value": "PAF93674D0B4E9963" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "ds", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": "", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{version=~\"^vmagent.*\"}, job)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(vm_app_version{version=~\"^vmagent.*\"}, job)", + "refId": "VictoriaMetrics-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{job=~\"$job\"}, instance)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(vm_app_version{job=~\"$job\"}, instance)", + "refId": "VictoriaMetrics-instance-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vmagent_remotewrite_requests_total{job=~\"$job\", instance=~\"$instance\"}, url)", + "description": "The remote write URLs", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "url", + "options": [], + "query": { + "query": "label_values(vmagent_remotewrite_requests_total{job=~\"$job\", instance=~\"$instance\"}, url)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "filters": [], + "hide": 0, + "name": "adhoc", + "skipUrlSync": false, + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "VictoriaMetrics - vmagent", + "uid": "G7Z9GzMGz", + "version": 1, + "weekStart": "", + "gnetId": 12683 + } +} \ No newline at end of file diff --git a/rootfs/usr/share/grafana/start-grafana b/rootfs/usr/share/grafana/start-grafana index 0ac48d1..e863aa1 100755 --- a/rootfs/usr/share/grafana/start-grafana +++ b/rootfs/usr/share/grafana/start-grafana @@ -123,6 +123,10 @@ if [ "on-cluster" != "${STORAGE_LOCATION}" ]; then echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "STORAGE"))')" > "${DCD}" fi +if [ "on-cluster" != "${PROMETHEUS_LOCATION}" ]; then + rm -rf "${DASHBOARD_LOCATION}"/drycc_victoriametrics_*.json +fi + for filename in "${DASHBOARD_LOCATION}"/*.json; do echo "Importing ${filename} ..." curl -i -XPOST --data "@${filename}" -H "${HEADER_ACCEPT}" -H "${HEADER_CONTENT_TYPE}" "http://${GRAFANA_USER}:${GRAFANA_PASSWD}@localhost:${GRAFANA_PORT}/api/dashboards/db" From 6d698adca997cf74ff8f8c14ccf87a2354392b94 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Mon, 19 May 2025 17:27:24 +0800 Subject: [PATCH 19/50] chore(grafana): change database to postgres --- charts/grafana/templates/_helpers.tmpl | 19 +++++++++ charts/grafana/values.yaml | 2 + rootfs/usr/share/grafana/grafana.ini.tpl | 50 ++---------------------- 3 files changed, 25 insertions(+), 46 deletions(-) diff --git a/charts/grafana/templates/_helpers.tmpl b/charts/grafana/templates/_helpers.tmpl index f6c6931..1e195ff 100644 --- a/charts/grafana/templates/_helpers.tmpl +++ b/charts/grafana/templates/_helpers.tmpl @@ -5,6 +5,25 @@ env: valueFrom: fieldRef: fieldPath: metadata.namespace +- name: GF_DATABASE_TYPE + value: postgres +{{- if (.Values.databaseUrl) }} +- name: GF_DATABASE_URL + value: "{{ .Values.databaseUrl }}" +{{- else if .Values.database.enabled }} +- name: GF_DATABASE_USER + valueFrom: + secretKeyRef: + name: database-creds + key: user +- name: GF_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: database-creds + key: password +- name: GF_DATABASE_URL + value: "postgres://$(GF_DATABASE_USER):$(GF_DATABASE_PASSWORD)@drycc-database.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:5432/grafana" +{{- end }} {{- if .Values.prometheusUrl }} - name: "PROMETHEUS_URL" value: "{{ .Values.prometheusUrl }}" diff --git a/charts/grafana/values.yaml b/charts/grafana/values.yaml index 87497e9..09ed45e 100644 --- a/charts/grafana/values.yaml +++ b/charts/grafana/values.yaml @@ -56,6 +56,8 @@ persistence: # this is usually a non required setting. environment: ALLOW_SIGN_UP: false +## databaseUrl and databaseReplicaUrl are will no longer use the built-in database component +databaseUrl: "" # The passportUrl, passportKey and passportSecret are will no longer use the built-in passport component passportUrl: "" passportKey: "" diff --git a/rootfs/usr/share/grafana/grafana.ini.tpl b/rootfs/usr/share/grafana/grafana.ini.tpl index 4274e96..03f712a 100644 --- a/rootfs/usr/share/grafana/grafana.ini.tpl +++ b/rootfs/usr/share/grafana/grafana.ini.tpl @@ -62,54 +62,12 @@ cert_key = {{ .CERT_KEY }} #################################### Database #################################### [database] # Either "mysql", "postgres" or "sqlite3", it's your choice -{{ if .DATABASE_TYPE }} -type = {{ .DATABASE_TYPE }} +{{ if .GF_DATABASE_TYPE }} +type = {{ .GF_DATABASE_TYPE }} {{ end }} -{{ if .DATABASE_HOST }} -host = {{ .DATABASE_HOST }} +{{ if .GF_DATABASE_URL }} +url = {{ .GF_DATABASE_URL }} {{ end }} -{{ if .DATABASE_NAME }} -name = {{ .DATABASE_NAME }} -{{ end }} -{{ if .DATABASE_USER }} -user = {{ .DATABASE_USER }} -{{ end }} -{{ if .DATABASE_PASSWORD }} -password = {{ .DATABASE_PASSWORD }} -{{ end }} - -# For "postgres" only, either "disable", "require" or "verify-full" -{{ if .DATABASE_SSL_MODE }} -ssl_mode = {{ .DATABASE_SSL_MODE }} -{{ end }} - -# For "sqlite3" only, path relative to data_path setting -{{ if .DATABASE_PATH }} -path = {{ .DATABASE_PATH }} -{{ end }} - -#################################### Session #################################### -[session] -# Either "memory", "file", "mysql", "postgres", default is "file" -provider = {{ default "file" .SESSION_PROVIDER }} - -# Provider config options -# memory: not have any config yet -# file: session dir path, is relative to grafana data_path -# mysql: go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name` -# postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable -{{ if .SESSION_PROVIDER_CONFIG }} -provider_config = {{ .SESSION_PROVIDER_CONFIG }} -{{ end }} - -# Session cookie name -cookie_name = {{ default "grafana_sess" .SESSION_COOKIE_NAME }} - -# If you use session in https only, default is false -cookie_secure = {{ default "false" .SESSION_COOKIE_SECURE }} - -# Session life time, default is 86400 -session_life_time = {{ default "86400" .SESSION_LIFE_TIME }} #################################### Analytics #################################### [analytics] From d5d8c59dec07b67842bc8ca14903e05d218a214a Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 20 May 2025 09:12:48 +0800 Subject: [PATCH 20/50] chore(grafana): bump grafana version --- rootfs/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index a3746bd..3b69504 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -4,7 +4,7 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/usr/share/grafana \ - GRAFANA_VERSION="11.6.0" \ + GRAFANA_VERSION="12.0.0" \ ENVTPL_VERSION="1.0.0" \ JQ_VERSION="1.7.1" From 31dc0776fb4c1cd9231d1cc3acc3a0aba1af6910 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 20 May 2025 11:23:22 +0800 Subject: [PATCH 21/50] chore(grafana): add database, registry and valkey dashboard --- .../api/dashboards/drycc_database.json | 1440 ++++++++++++++++ .../api/dashboards/drycc_registry.json | 1475 +++++++++++++++++ .../grafana/api/dashboards/drycc_valkey.json | 1382 +++++++++++++++ rootfs/usr/share/grafana/start-grafana | 4 +- 4 files changed, 4300 insertions(+), 1 deletion(-) create mode 100644 rootfs/usr/share/grafana/api/dashboards/drycc_database.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/drycc_registry.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/drycc_valkey.json diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_database.json b/rootfs/usr/share/grafana/api/dashboards/drycc_database.json new file mode 100644 index 0000000..3b2112b --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_database.json @@ -0,0 +1,1440 @@ +{ + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A quickstart to setup the Prometheus PostgreSQL Exporter with preconfigured dashboards, alerting rules, and recording rules.", + "editable": true, + "gnetId": 14114, + "graphTooltip": 0, + "id": 1, + "iteration": 1603191461722, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 20, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 1, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_fetched{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "fetched", + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_fetched" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_returned{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "returned", + "measurement": "postgresql", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_fetched" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_inserted{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "inserted", + "measurement": "postgresql", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_fetched" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_updated{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "updated", + "measurement": "postgresql", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_fetched" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_deleted{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "deleted", + "measurement": "postgresql", + "policy": "default", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_fetched" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rows", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "height": "55px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_xact_commit{datname=~\"$db\",instance=~\"$instance\"}[5m])) + sum(irate(pg_stat_database_xact_rollback{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "xact_commit" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 1800, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + } + ], + "thresholds": "", + "title": "QPS", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 1, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 2, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance=~'$instance'}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_alloc", + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_alloc" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{instance=~'$instance'}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_backend_fsync", + "measurement": "postgresql", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_alloc" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_backend{instance=~'$instance'}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_backend", + "measurement": "postgresql", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_alloc" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_clean{instance=~'$instance'}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_clean", + "measurement": "postgresql", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_alloc" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{instance=~'$instance'}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_checkpoint", + "measurement": "postgresql", + "policy": "default", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_alloc" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Buffers", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 3, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "conflicts", + "dsType": "prometheus", + "expr": "sum(rate(pg_stat_database_deadlocks{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "deadlocks", + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "conflicts" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "deadlocks", + "dsType": "prometheus", + "expr": "sum(rate(pg_stat_database_conflicts{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "conflicts", + "measurement": "postgresql", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "deadlocks" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Conflicts/Deadlocks", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 12, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": true, + "pluginVersion": "7.2.1", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=~\"$instance\"}) / (sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=~\"$instance\"}) + sum(pg_stat_database_blks_read{datname=~\"$db\",instance=~\"$instance\"}))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "cache hit rate", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache hit ratio", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 13, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_stat_database_numbackends{datname=~\"$db\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{__name__}}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of active connections", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [ + "postgres" + ], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(up{job=~\"postgres.*\"},instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", + "hide": 0, + "includeAll": true, + "label": "db", + "multi": false, + "name": "db", + "options": [], + "query": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "Postgres Overview", + "value": "Postgres Overview" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "postgres", + "value": "postgres" + }, + "datasource": "$datasource", + "definition": "label_values(pg_up, job)", + "hide": 0, + "includeAll": false, + "label": "job", + "multi": false, + "name": "job", + "options": [ + { + "selected": true, + "text": "postgres", + "value": "postgres" + } + ], + "query": "label_values(pg_up, job)", + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "PostgreSQL Exporter Quickstart and Dashboard", + "uid": "wGgaPlciz", + "version": 5 + } +} \ No newline at end of file diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_registry.json b/rootfs/usr/share/grafana/api/dashboards/drycc_registry.json new file mode 100644 index 0000000..5a8105c --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_registry.json @@ -0,0 +1,1475 @@ +{ + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": 9621, + "graphTooltip": 0, + "id": null, + "iteration": 1547215355938, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 22, + "panels": [], + "title": "General Counters, CPU, Memory and File Descriptor Stats", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_DS_PROMETHEUS}", + "description": "start time of the process", + "format": "dateTimeFromNow", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "110%", + "prefix": "", + "prefixFontSize": "110%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "process_start_time_seconds{app=\"$app\", instance=\"$instance\"} * 1000", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Start Time", + "type": "singlestat", + "valueFontSize": "110%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#967302", + "#967302", + "#967302" + ], + "datasource": "${DS_DS_PROMETHEUS}", + "description": "The number of cache request received", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 4, + "y": 1 + }, + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "150%", + "prefix": "", + "prefixFontSize": "150%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "#eab839", + "full": false, + "lineColor": "#e5ac0e", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "registry_storage_cache_total{instance=\"$instance\",type=\"Request\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ type }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Total Cache Requests", + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#1f78c1", + "#1f78c1", + "#1f78c1" + ], + "datasource": "${DS_DS_PROMETHEUS}", + "description": "The number of cache request received (hits)", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 9, + "y": 1 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "150%", + "prefix": "", + "prefixFontSize": "150%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "#5195ce", + "full": false, + "lineColor": "#0a50a1", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "registry_storage_cache_total{instance=\"$instance\",type=\"Hit\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ type }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Total Hit Cache Requests", + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#890f02", + "#890f02", + "#890f02" + ], + "datasource": "${DS_DS_PROMETHEUS}", + "description": "The number of cache request received (miss)", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 14, + "y": 1 + }, + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "150%", + "prefix": "", + "prefixFontSize": "150%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "#e24d42", + "full": false, + "lineColor": "#e24d42", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "registry_storage_cache_total{instance=\"$instance\",type=\"Miss\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ type }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Total Missed Cache Requests", + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_DS_PROMETHEUS}", + "description": "Process Resident Memory Usage", + "format": "decbytes", + "gauge": { + "maxValue": 1000, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 19, + "y": 1 + }, + "id": 24, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "150%", + "prefix": "", + "prefixFontSize": "150%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(process_resident_memory_bytes{instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Resident Memory Usage", + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_DS_PROMETHEUS}", + "description": "Average user and system CPU time spent in seconds.", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(process_cpu_seconds_total{instance=\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "CPU Time", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_DS_PROMETHEUS}", + "description": "Virtual and Resident memory size in bytes, averages over 5 min interval", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 4 + }, + "id": 32, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(process_resident_memory_bytes{instance=\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Resident Mem", + "refId": "A" + }, + { + "expr": "avg(rate(process_virtual_memory_bytes{instance=\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Virtual Mem", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_DS_PROMETHEUS}", + "description": "Number of open file descriptors", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_open_fds{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Open FD", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Open File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 20, + "panels": [], + "title": "Registry Metrics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_DS_PROMETHEUS}", + "description": "The HTTP requests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 26, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 3, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(rate(registry_http_requests_total{instance=\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ handler }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "HTTP Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_DS_PROMETHEUS}", + "description": "The in-flight HTTP requests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 12 + }, + "id": 18, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "registry_http_in_flight_requests{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ handler }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "In-Flight HTTP Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_DS_PROMETHEUS}", + "description": "The rate of registry cache requests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 28, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 3, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(rate(registry_storage_cache_total{instance=\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Registry Cache Requests Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "increase(registry_storage_action_seconds_sum{instance=\"$instance\"}[2m]) * 1000", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{ action }}", + "refId": "A", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Registry Action Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "${DS_DS_PROMETHEUS}", + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 20 + }, + "heatmap": {}, + "highlightCards": true, + "id": 36, + "legend": { + "show": false + }, + "links": [], + "targets": [ + { + "expr": "rate(registry_http_request_duration_seconds_bucket{handler=\"blob\"}[10m])", + "format": "heatmap", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "HTTP Request Latencies in seconds (blob)", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": "0", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "${DS_DS_PROMETHEUS}", + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 20 + }, + "heatmap": {}, + "highlightCards": true, + "id": 38, + "legend": { + "show": false + }, + "links": [], + "targets": [ + { + "expr": "rate(registry_http_request_duration_seconds_bucket{handler=\"base\"}[10m])", + "format": "heatmap", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Upload HTTP Request Latencies in seconds (base)", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": "0", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "${DS_DS_PROMETHEUS}", + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 27 + }, + "heatmap": {}, + "highlightCards": true, + "id": 2, + "legend": { + "show": false + }, + "links": [], + "targets": [ + { + "expr": "rate(registry_http_request_duration_seconds_bucket{handler=\"blob_upload\"}[10m])", + "format": "heatmap", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Upload HTTP Request Latencies in seconds (blob_upload)", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": "0", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "${DS_DS_PROMETHEUS}", + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 27 + }, + "heatmap": {}, + "highlightCards": true, + "id": 42, + "legend": { + "show": false + }, + "links": [], + "targets": [ + { + "expr": "rate(registry_http_request_duration_seconds_bucket{handler=\"manifest\"}[10m])", + "format": "heatmap", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Manifest HTTP Request Latencies in seconds (manifest)", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": "0", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "${DS_DS_PROMETHEUS}", + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 27 + }, + "heatmap": {}, + "highlightCards": true, + "id": 40, + "legend": { + "show": false + }, + "links": [], + "targets": [ + { + "expr": "rate(registry_http_request_duration_seconds_bucket{handler=\"catalog\"}[10m])", + "format": "heatmap", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Catalog HTTP Request Latencies in seconds (catalog)", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": "0", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + } + ], + "refresh": "10s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "docker", + "registry", + "internals" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "${DS_DS_PROMETHEUS}", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [], + "query": "query_result(registry_http_in_flight_requests)", + "refresh": 2, + "regex": "/.*kubernetes_namespace=\"([^\"]+).*/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_DS_PROMETHEUS}", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "app", + "options": [], + "query": "query_result(registry_http_in_flight_requests{kubernetes_namespace=\"$namespace\"})", + "refresh": 2, + "regex": "/.*app=\"([^\"]+)/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_DS_PROMETHEUS}", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "query_result(up{app=\"$app\"})", + "refresh": 1, + "regex": "/.*instance=\"([^\"]+).*/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Docker Registry", + "uid": "CoBSgj8iz", + "version": 19 + } +} \ No newline at end of file diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_valkey.json b/rootfs/usr/share/grafana/api/dashboards/drycc_valkey.json new file mode 100644 index 0000000..fbca559 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_valkey.json @@ -0,0 +1,1382 @@ +{ + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A quickstart to setup the Prometheus Redis Exporter with preconfigured dashboards, alerting rules, and recording rules.", + "editable": true, + "gnetId": 14091, + "graphTooltip": 1, + "id": 24, + "iteration": 1602758020790, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "Performance", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average taken across instances", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 18, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(redis_commands_total{instance=~\"$instance\"} [1m])) by (cmd)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{cmd}}", + "metric": "redis_command_calls_total", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Commands per second", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average taken across instances", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 1 + }, + "hiddenSeries": false, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(redis_commands_duration_seconds_total{instance=~\"$instance\"}[1m])) by (cmd)\n /\navg(irate(redis_commands_total{instance=~\"$instance\"}[1m])) by (cmd)\n", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ cmd }}", + "metric": "redis_command_calls_total", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Command latency per second", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Hit ratio": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "description": "Hit rate shows the percentage of key space lookups that hit a key.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 1 + }, + "hiddenSeries": false, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Target/", + "color": "#56A64B", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "linewidth": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(redis_keyspace_hits_total{instance=~\"$instance\"}[1m]) / (irate(redis_keyspace_misses_total{instance=~\"$instance\"}[1m]) + irate(redis_keyspace_hits_total{instance=~\"$instance\"}[1m]))) by (instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "", + "refId": "A", + "step": 240, + "target": "" + }, + { + "expr": "1", + "interval": "", + "legendFormat": "Target hit ratio for cache", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Hit ratio per instance", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 22, + "panels": [], + "title": "Memory", + "type": "row" + }, + { + "aliasColors": { + "max": "#BF1B00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total taken across instances", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 7, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/max/", + "color": "#E02F44", + "dashes": true, + "fill": 0, + "linewidth": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(redis_memory_used_bytes{instance=~\"$instance\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Used Memory", + "metric": "", + "refId": "A", + "step": 240, + "target": "" + }, + { + "expr": "sum(redis_memory_max_bytes{instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Configured max memory", + "refId": "B", + "step": 240 + }, + { + "expr": "sum(redis_memory_used_rss_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Used RSS memory", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Recommend restart redis": "red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 9 + }, + "hiddenSeries": false, + "id": 10, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/restart/", + "color": "#E02F44", + "dashes": true, + "fill": 0, + "linewidth": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "redis_memory_fragmentation_ratio{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory fragmentation ratio per instance", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Evictions": "red", + "evicts": "#890F02", + "memcached_items_evicted_total{instance=\"172.17.0.1:9150\",job=\"prometheus\"}": "#890F02", + "reclaims": "#3F6833", + "{container=\"redis-exporter\", instance=\"redis-86cb5d76d7-fcdln:redis-exporter:redis-metrics\", job=\"default/redis\", namespace=\"default\", pod=\"redis-86cb5d76d7-fcdln\"}": "red", + "{instance=\"redis-86cb5d76d7-fcdln:redis-exporter:redis-metrics\"}": "red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 9 + }, + "hiddenSeries": false, + "id": 8, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "reclaims", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(redis_evicted_keys_total{instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Key evictions per second per instance", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 26, + "panels": [], + "title": "Basic activity", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Sum taken across instances", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(redis_connected_clients{instance=~\"$instance\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Connected", + "refId": "A" + }, + { + "expr": "sum(redis_blocked_clients{instance=~\"$instance\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Blocked", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Connected/Blocked Clients", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "db1": "yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Sum taken across instances", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 17 + }, + "hiddenSeries": false, + "id": 5, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum (redis_db_keys{instance=~\"$instance\"}) by (db)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ db }}", + "refId": "A", + "step": 240, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Items per DB", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Sum taken across instances", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 17 + }, + "hiddenSeries": false, + "id": 13, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum (redis_db_keys{instance=~\"$instance\"}) - sum (redis_db_keys_expiring{instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Not expiring", + "refId": "A", + "step": 240, + "target": "" + }, + { + "expr": "sum(redis_db_keys_expiring{instance=~\"$instance\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Expiring", + "metric": "", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Expiring vs Not-Expiring Keys", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "This metric will only be non-zero if the instance is a master", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(redis_connected_slaves{instance=~\"$instance\"}) by (instance)", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Connected slaves by instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "This metric is only exported if the instance is a slave.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 24 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "redis_master_last_io_seconds_ago{instance=~\"$instance\"}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Time since last master connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 25, + "style": "dark", + "tags": [ + "prometheus", + "redis" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "tags": [], + "text": "redisdb-7d6b98cd98-kjt5x + redisdb-7d6b98cd98-pbkg6", + "value": [ + "redisdb-7d6b98cd98-kjt5x", + "redisdb-7d6b98cd98-pbkg6" + ] + }, + "datasource": "${datasource}", + "definition": "label_values(redis_up, instance)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(redis_up, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Redis Exporter Quickstart and Dashboard", + "uid": "bRd48yKMdd", + "version": 5 + } +} \ No newline at end of file diff --git a/rootfs/usr/share/grafana/start-grafana b/rootfs/usr/share/grafana/start-grafana index e863aa1..038f4de 100755 --- a/rootfs/usr/share/grafana/start-grafana +++ b/rootfs/usr/share/grafana/start-grafana @@ -99,11 +99,12 @@ DCD="${DASHBOARD_LOCATION}"/drycc_component_health.json # remove off-cluster component panel if [ "on-cluster" != "${VALKEY_LOCATION}" ]; then - rm -rf "${DASHBOARD_LOCATION}"/valkey.json + rm -rf "${DASHBOARD_LOCATION}"/drycc_valkey.json # shellcheck disable=SC2005 echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "VALKEY"))')" > "${DCD}" fi if [ "on-cluster" != "${DATABASE_LOCATION}" ]; then + rm -rf "${DASHBOARD_LOCATION}"/drycc_database.json # shellcheck disable=SC2005 echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "DATABASE"))')" > "${DCD}" fi @@ -112,6 +113,7 @@ if [ "on-cluster" != "${PASSPORT_LOCATION}" ]; then echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "PASSPORT"))')" > "${DCD}" fi if [ "on-cluster" != "${REGISTRY_LOCATION}" ]; then + rm -rf "${DASHBOARD_LOCATION}"/drycc_registry.json # shellcheck disable=SC2005 echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "REGISTRY"))')" > "${DCD}" # shellcheck disable=SC2005 From b17ed4e444feca8dccc3eaddb558284f2c11f25c Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 20 May 2025 15:22:37 +0800 Subject: [PATCH 22/50] chore(charts): add initcontainer --- .../grafana/templates/grafana-deployment.yaml | 10 + .../dashboards/drycc_component_health.json | 308 +- .../api/dashboards/drycc_database.json | 1168 ++-- .../api/dashboards/drycc_registry.json | 73 +- .../grafana/api/dashboards/drycc_storage.json | 94 +- .../grafana/api/dashboards/drycc_valkey.json | 45 +- .../drycc_victoriametrics_cluster.json | 486 +- .../drycc_victoriametrics_vmagent.json | 338 +- .../grafana/api/dashboards/kubernetes.json | 5929 +++++++++++++++++ .../api/dashboards/kubernetes_apiserver.json | 642 ++ .../api/dashboards/kubernetes_cluster.json | 2013 ++++++ .../kubernetes_deployment_metrics.json | 73 +- .../api/dashboards/kubernetes_metrics.json | 3393 ++++++++++ rootfs/usr/share/grafana/grafana.ini.tpl | 4 +- rootfs/usr/share/grafana/start-grafana | 2 +- 15 files changed, 12810 insertions(+), 1768 deletions(-) create mode 100644 rootfs/usr/share/grafana/api/dashboards/kubernetes.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/kubernetes_apiserver.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/kubernetes_cluster.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/kubernetes_metrics.json diff --git a/charts/grafana/templates/grafana-deployment.yaml b/charts/grafana/templates/grafana-deployment.yaml index 846db71..b4fb98b 100644 --- a/charts/grafana/templates/grafana-deployment.yaml +++ b/charts/grafana/templates/grafana-deployment.yaml @@ -29,6 +29,16 @@ spec: fsGroup: 1001 runAsGroup: 1001 runAsUser: 1001 + initContainers: + - name: drycc-grafana-init + image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest + imagePullPolicy: {{.Values.imagePullPolicy}} + args: + - netcat + - -v + - -u + - $(GF_DATABASE_URL) + {{- include "grafana.envs" . | indent 8 }} containers: - name: drycc-grafana image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/grafana:{{.Values.imageTag}} diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json b/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json index 453626a..b8046bd 100644 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json @@ -99,21 +99,15 @@ "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",container!=\"\",namespace=\"drycc\"}[1m])) * 1000", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "namespace" - ], + "params": ["namespace"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -208,21 +202,15 @@ "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",container!=\"\",namespace=\"drycc\"})", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "namespace" - ], + "params": ["namespace"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -327,21 +315,15 @@ "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",container=~\"^drycc-passport.*$\"}[1m])) by (pod)", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -438,21 +420,15 @@ "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-passport.*$\"}) by (pod)", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -566,21 +542,15 @@ "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-controller.*$\"}[1m])) by (pod) * 1000", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -677,21 +647,15 @@ "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-controller.*$\"}) by (pod)", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -805,21 +769,15 @@ "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-builder.*$\"}[1m])) by (pod) * 1000", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -915,21 +873,15 @@ "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-builder.*$\"}) by (pod)", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -1043,21 +995,15 @@ "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-database.*$\"}[1m])) by (pod) * 1000", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -1152,21 +1098,15 @@ "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-database.*$\"}) by (pod)", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -1280,21 +1220,15 @@ "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-logger-fluentbit.*$\"}[1m])) by (pod) * 1000", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -1389,21 +1323,15 @@ "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-logger-fluentbit.*$\"}) by (pod)", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -1519,21 +1447,15 @@ "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-grafana.*$\"}[1m])) by (pod) * 1000", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -1630,21 +1552,15 @@ "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-grafana.*$\"}) by (pod)", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -1758,21 +1674,15 @@ "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=\"drycc-logger\"}[1m])) by (pod) * 1000", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -1867,21 +1777,15 @@ "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=\"drycc-logger\"}) by (pod)", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -1995,21 +1899,15 @@ "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage.*$\"}[1m])) by (pod) * 1000", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -2104,21 +2002,15 @@ "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage.*$\"}) by (pod)", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -2232,21 +2124,15 @@ "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-valkey.*$\"}[1m])) by (pod) * 1000", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -2340,21 +2226,15 @@ "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-valkey.*$\"}) by (pod)", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -2468,21 +2348,15 @@ "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-registry.*$\"}[1m])) by (pod) * 1000", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -2577,21 +2451,15 @@ "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=\"drycc-registry\"}) by (pod)", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -2705,21 +2573,15 @@ "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-registry-proxy.*$\"}[1m])) by (pod) * 1000", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -2814,21 +2676,15 @@ "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-registry-proxy.*$\"}) by (pod)", "groupBy": [ { - "params": [ - "$interval" - ], + "params": ["$interval"], "type": "time" }, { - "params": [ - "pod_name" - ], + "params": ["pod_name"], "type": "tag" }, { - "params": [ - "null" - ], + "params": ["null"], "type": "fill" } ], @@ -2884,9 +2740,7 @@ "refresh": "5s", "schemaVersion": 37, "style": "dark", - "tags": [ - "drycc" - ], + "tags": ["drycc"], "templating": { "list": [] }, @@ -2907,17 +2761,7 @@ "2h", "1d" ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] }, "timezone": "browser", "title": "Drycc Component Health", @@ -2925,5 +2769,5 @@ "version": 1, "weekStart": "" }, - "overwrite": false -} \ No newline at end of file + "overwrite": true +} diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_database.json b/rootfs/usr/share/grafana/api/dashboards/drycc_database.json index 3b2112b..bf40df8 100644 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_database.json +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_database.json @@ -1,17 +1,17 @@ { "dashboard": { "annotations": { - "list": [ + "list": [ { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" } - ] + ] }, "description": "A quickstart to setup the Prometheus PostgreSQL Exporter with preconfigured dashboards, alerting rules, and recording rules.", "editable": true, @@ -21,49 +21,48 @@ "iteration": 1603191461722, "links": [], "panels": [ - { + { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { - "defaults": { + "defaults": { "custom": {} - }, - "overrides": [] + }, + "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { - "h": 7, - "w": 20, - "x": 0, - "y": 0 + "h": 7, + "w": 20, + "x": 0, + "y": 0 }, "hiddenSeries": false, "id": 1, "isNew": true, "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": false, - "values": true + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { - "alertThreshold": true + "alertThreshold": true }, "percentage": false, "pluginVersion": "7.2.1", @@ -75,24 +74,20 @@ "stack": false, "steppedLine": false, "targets": [ - { + { "alias": "fetched", "dsType": "prometheus", "expr": "sum(irate(pg_stat_database_tup_fetched{datname=~\"$db\",instance=~\"$instance\"}[5m]))", "format": "time_series", "groupBy": [ - { - "params": [ - "$interval" - ], + { + "params": ["$interval"], "type": "time" - }, - { - "params": [ - "null" - ], + }, + { + "params": ["null"], "type": "fill" - } + } ], "intervalFactor": 2, "legendFormat": "fetched", @@ -101,52 +96,44 @@ "refId": "A", "resultFormat": "time_series", "select": [ - [ + [ { - "params": [ - "tup_fetched" - ], - "type": "field" + "params": ["tup_fetched"], + "type": "field" }, { - "params": [], - "type": "mean" + "params": [], + "type": "mean" }, { - "params": [ - "10s" - ], - "type": "non_negative_derivative" + "params": ["10s"], + "type": "non_negative_derivative" } - ] + ] ], "step": 120, "tags": [ - { + { "key": "instance", "operator": "=~", "value": "/^$instance$/" - } + } ] - }, - { + }, + { "alias": "fetched", "dsType": "prometheus", "expr": "sum(irate(pg_stat_database_tup_returned{datname=~\"$db\",instance=~\"$instance\"}[5m]))", "format": "time_series", "groupBy": [ - { - "params": [ - "$interval" - ], + { + "params": ["$interval"], "type": "time" - }, - { - "params": [ - "null" - ], + }, + { + "params": ["null"], "type": "fill" - } + } ], "intervalFactor": 2, "legendFormat": "returned", @@ -155,52 +142,44 @@ "refId": "B", "resultFormat": "time_series", "select": [ - [ + [ { - "params": [ - "tup_fetched" - ], - "type": "field" + "params": ["tup_fetched"], + "type": "field" }, { - "params": [], - "type": "mean" + "params": [], + "type": "mean" }, { - "params": [ - "10s" - ], - "type": "non_negative_derivative" + "params": ["10s"], + "type": "non_negative_derivative" } - ] + ] ], "step": 120, "tags": [ - { + { "key": "instance", "operator": "=~", "value": "/^$instance$/" - } + } ] - }, - { + }, + { "alias": "fetched", "dsType": "prometheus", "expr": "sum(irate(pg_stat_database_tup_inserted{datname=~\"$db\",instance=~\"$instance\"}[5m]))", "format": "time_series", "groupBy": [ - { - "params": [ - "$interval" - ], + { + "params": ["$interval"], "type": "time" - }, - { - "params": [ - "null" - ], + }, + { + "params": ["null"], "type": "fill" - } + } ], "intervalFactor": 2, "legendFormat": "inserted", @@ -209,52 +188,44 @@ "refId": "C", "resultFormat": "time_series", "select": [ - [ + [ { - "params": [ - "tup_fetched" - ], - "type": "field" + "params": ["tup_fetched"], + "type": "field" }, { - "params": [], - "type": "mean" + "params": [], + "type": "mean" }, { - "params": [ - "10s" - ], - "type": "non_negative_derivative" + "params": ["10s"], + "type": "non_negative_derivative" } - ] + ] ], "step": 120, "tags": [ - { + { "key": "instance", "operator": "=~", "value": "/^$instance$/" - } + } ] - }, - { + }, + { "alias": "fetched", "dsType": "prometheus", "expr": "sum(irate(pg_stat_database_tup_updated{datname=~\"$db\",instance=~\"$instance\"}[5m]))", "format": "time_series", "groupBy": [ - { - "params": [ - "$interval" - ], + { + "params": ["$interval"], "type": "time" - }, - { - "params": [ - "null" - ], + }, + { + "params": ["null"], "type": "fill" - } + } ], "intervalFactor": 2, "legendFormat": "updated", @@ -263,52 +234,44 @@ "refId": "D", "resultFormat": "time_series", "select": [ - [ + [ { - "params": [ - "tup_fetched" - ], - "type": "field" + "params": ["tup_fetched"], + "type": "field" }, { - "params": [], - "type": "mean" + "params": [], + "type": "mean" }, { - "params": [ - "10s" - ], - "type": "non_negative_derivative" + "params": ["10s"], + "type": "non_negative_derivative" } - ] + ] ], "step": 120, "tags": [ - { + { "key": "instance", "operator": "=~", "value": "/^$instance$/" - } + } ] - }, - { + }, + { "alias": "fetched", "dsType": "prometheus", "expr": "sum(irate(pg_stat_database_tup_deleted{datname=~\"$db\",instance=~\"$instance\"}[5m]))", "format": "time_series", "groupBy": [ - { - "params": [ - "$interval" - ], + { + "params": ["$interval"], "type": "time" - }, - { - "params": [ - "null" - ], + }, + { + "params": ["null"], "type": "fill" - } + } ], "intervalFactor": 2, "legendFormat": "deleted", @@ -317,34 +280,30 @@ "refId": "E", "resultFormat": "time_series", "select": [ - [ + [ { - "params": [ - "tup_fetched" - ], - "type": "field" + "params": ["tup_fetched"], + "type": "field" }, { - "params": [], - "type": "mean" + "params": [], + "type": "mean" }, { - "params": [ - "10s" - ], - "type": "non_negative_derivative" + "params": ["10s"], + "type": "non_negative_derivative" } - ] + ] ], "step": 120, "tags": [ - { + { "key": "instance", "operator": "=~", "value": "/^$instance$/" - } + } ] - } + } ], "thresholds": [], "timeFrom": null, @@ -352,74 +311,73 @@ "timeShift": null, "title": "Rows", "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" }, "type": "graph", "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] }, "yaxes": [ - { + { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true - }, - { + }, + { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true - } + } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false, + "alignLevel": null } - }, - { + }, + { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" ], - "datasource": "$datasource", "decimals": 0, "editable": true, "error": false, "fieldConfig": { - "defaults": { + "defaults": { "custom": {} - }, - "overrides": [] + }, + "overrides": [] }, "format": "none", "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true }, "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 0 + "h": 3, + "w": 4, + "x": 20, + "y": 0 }, "height": "55px", "id": 11, @@ -428,14 +386,14 @@ "links": [], "mappingType": 1, "mappingTypes": [ - { + { "name": "value to text", "value": 1 - }, - { + }, + { "name": "range to text", "value": 2 - } + } ], "maxDataPoints": 100, "nullPointMode": "connected", @@ -445,37 +403,33 @@ "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ - { + { "from": "null", "text": "N/A", "to": "null" - } + } ], "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true }, "tableColumn": "", "targets": [ - { + { "dsType": "prometheus", "expr": "sum(irate(pg_stat_database_xact_commit{datname=~\"$db\",instance=~\"$instance\"}[5m])) + sum(irate(pg_stat_database_xact_rollback{datname=~\"$db\",instance=~\"$instance\"}[5m]))", "format": "time_series", "groupBy": [ - { - "params": [ - "$interval" - ], + { + "params": ["$interval"], "type": "time" - }, - { - "params": [ - "null" - ], + }, + { + "params": ["null"], "type": "fill" - } + } ], "intervalFactor": 2, "measurement": "postgresql", @@ -483,34 +437,30 @@ "refId": "A", "resultFormat": "time_series", "select": [ - [ + [ { - "params": [ - "xact_commit" - ], - "type": "field" + "params": ["xact_commit"], + "type": "field" }, { - "params": [], - "type": "mean" + "params": [], + "type": "mean" }, { - "params": [ - "10s" - ], - "type": "non_negative_derivative" + "params": ["10s"], + "type": "non_negative_derivative" } - ] + ] ], "step": 1800, "tags": [ - { + { "key": "instance", "operator": "=~", "value": "/^$instance$/" - } + } ] - } + } ], "thresholds": "", "title": "QPS", @@ -518,59 +468,58 @@ "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ - { + { "op": "=", "text": "N/A", "value": "null" - } + } ], "valueName": "avg" - }, - { + }, + { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", "decimals": 1, "editable": true, "error": false, "fieldConfig": { - "defaults": { + "defaults": { "custom": {} - }, - "overrides": [] + }, + "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 7 + "h": 7, + "w": 12, + "x": 0, + "y": 7 }, "hiddenSeries": false, "id": 2, "isNew": true, "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "hideZero": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { - "alertThreshold": true + "alertThreshold": true }, "percentage": false, "pluginVersion": "7.2.1", @@ -582,24 +531,20 @@ "stack": false, "steppedLine": false, "targets": [ - { + { "alias": "Buffers Allocated", "dsType": "prometheus", "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance=~'$instance'}[5m])", "format": "time_series", "groupBy": [ - { - "params": [ - "$interval" - ], + { + "params": ["$interval"], "type": "time" - }, - { - "params": [ - "null" - ], + }, + { + "params": ["null"], "type": "fill" - } + } ], "intervalFactor": 2, "legendFormat": "buffers_alloc", @@ -608,50 +553,44 @@ "refId": "A", "resultFormat": "time_series", "select": [ - [ + [ { - "params": [ - "buffers_alloc" - ], - "type": "field" + "params": ["buffers_alloc"], + "type": "field" }, { - "params": [], - "type": "mean" + "params": [], + "type": "mean" }, { - "params": [], - "type": "difference" + "params": [], + "type": "difference" } - ] + ] ], "step": 240, "tags": [ - { + { "key": "instance", "operator": "=~", "value": "/^$instance$/" - } + } ] - }, - { + }, + { "alias": "Buffers Allocated", "dsType": "prometheus", "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{instance=~'$instance'}[5m])", "format": "time_series", "groupBy": [ - { - "params": [ - "$interval" - ], + { + "params": ["$interval"], "type": "time" - }, - { - "params": [ - "null" - ], + }, + { + "params": ["null"], "type": "fill" - } + } ], "intervalFactor": 2, "legendFormat": "buffers_backend_fsync", @@ -660,50 +599,44 @@ "refId": "B", "resultFormat": "time_series", "select": [ - [ + [ { - "params": [ - "buffers_alloc" - ], - "type": "field" + "params": ["buffers_alloc"], + "type": "field" }, { - "params": [], - "type": "mean" + "params": [], + "type": "mean" }, { - "params": [], - "type": "difference" + "params": [], + "type": "difference" } - ] + ] ], "step": 240, "tags": [ - { + { "key": "instance", "operator": "=~", "value": "/^$instance$/" - } + } ] - }, - { + }, + { "alias": "Buffers Allocated", "dsType": "prometheus", "expr": "irate(pg_stat_bgwriter_buffers_backend{instance=~'$instance'}[5m])", "format": "time_series", "groupBy": [ - { - "params": [ - "$interval" - ], + { + "params": ["$interval"], "type": "time" - }, - { - "params": [ - "null" - ], + }, + { + "params": ["null"], "type": "fill" - } + } ], "intervalFactor": 2, "legendFormat": "buffers_backend", @@ -712,50 +645,44 @@ "refId": "C", "resultFormat": "time_series", "select": [ - [ + [ { - "params": [ - "buffers_alloc" - ], - "type": "field" + "params": ["buffers_alloc"], + "type": "field" }, { - "params": [], - "type": "mean" + "params": [], + "type": "mean" }, { - "params": [], - "type": "difference" + "params": [], + "type": "difference" } - ] + ] ], "step": 240, "tags": [ - { + { "key": "instance", "operator": "=~", "value": "/^$instance$/" - } + } ] - }, - { + }, + { "alias": "Buffers Allocated", "dsType": "prometheus", "expr": "irate(pg_stat_bgwriter_buffers_clean{instance=~'$instance'}[5m])", "format": "time_series", "groupBy": [ - { - "params": [ - "$interval" - ], + { + "params": ["$interval"], "type": "time" - }, - { - "params": [ - "null" - ], + }, + { + "params": ["null"], "type": "fill" - } + } ], "intervalFactor": 2, "legendFormat": "buffers_clean", @@ -764,50 +691,44 @@ "refId": "D", "resultFormat": "time_series", "select": [ - [ + [ { - "params": [ - "buffers_alloc" - ], - "type": "field" + "params": ["buffers_alloc"], + "type": "field" }, { - "params": [], - "type": "mean" + "params": [], + "type": "mean" }, { - "params": [], - "type": "difference" + "params": [], + "type": "difference" } - ] + ] ], "step": 240, "tags": [ - { + { "key": "instance", "operator": "=~", "value": "/^$instance$/" - } + } ] - }, - { + }, + { "alias": "Buffers Allocated", "dsType": "prometheus", "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{instance=~'$instance'}[5m])", "format": "time_series", "groupBy": [ - { - "params": [ - "$interval" - ], + { + "params": ["$interval"], "type": "time" - }, - { - "params": [ - "null" - ], + }, + { + "params": ["null"], "type": "fill" - } + } ], "intervalFactor": 2, "legendFormat": "buffers_checkpoint", @@ -816,32 +737,30 @@ "refId": "E", "resultFormat": "time_series", "select": [ - [ + [ { - "params": [ - "buffers_alloc" - ], - "type": "field" + "params": ["buffers_alloc"], + "type": "field" }, { - "params": [], - "type": "mean" + "params": [], + "type": "mean" }, { - "params": [], - "type": "difference" + "params": [], + "type": "difference" } - ] + ] ], "step": 240, "tags": [ - { + { "key": "instance", "operator": "=~", "value": "/^$instance$/" - } + } ] - } + } ], "thresholds": [], "timeFrom": null, @@ -849,83 +768,82 @@ "timeShift": null, "title": "Buffers", "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" }, "type": "graph", "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] }, "yaxes": [ - { + { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true - }, - { + }, + { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true - } + } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false, + "alignLevel": null } - }, - { + }, + { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { - "defaults": { + "defaults": { "custom": {} - }, - "overrides": [] + }, + "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 7 + "h": 7, + "w": 12, + "x": 12, + "y": 7 }, "hiddenSeries": false, "id": 3, "isNew": true, "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { - "alertThreshold": true + "alertThreshold": true }, "percentage": false, "pluginVersion": "7.2.1", @@ -937,24 +855,20 @@ "stack": false, "steppedLine": false, "targets": [ - { + { "alias": "conflicts", "dsType": "prometheus", "expr": "sum(rate(pg_stat_database_deadlocks{datname=~\"$db\",instance=~\"$instance\"}[5m]))", "format": "time_series", "groupBy": [ - { - "params": [ - "$interval" - ], + { + "params": ["$interval"], "type": "time" - }, - { - "params": [ - "null" - ], + }, + { + "params": ["null"], "type": "fill" - } + } ], "intervalFactor": 2, "legendFormat": "deadlocks", @@ -963,50 +877,44 @@ "refId": "A", "resultFormat": "time_series", "select": [ - [ + [ { - "params": [ - "conflicts" - ], - "type": "field" + "params": ["conflicts"], + "type": "field" }, { - "params": [], - "type": "mean" + "params": [], + "type": "mean" }, { - "params": [], - "type": "difference" + "params": [], + "type": "difference" } - ] + ] ], "step": 240, "tags": [ - { + { "key": "instance", "operator": "=~", "value": "/^$instance$/" - } + } ] - }, - { + }, + { "alias": "deadlocks", "dsType": "prometheus", "expr": "sum(rate(pg_stat_database_conflicts{datname=~\"$db\",instance=~\"$instance\"}[5m]))", "format": "time_series", "groupBy": [ - { - "params": [ - "$interval" - ], + { + "params": ["$interval"], "type": "time" - }, - { - "params": [ - "null" - ], + }, + { + "params": ["null"], "type": "fill" - } + } ], "intervalFactor": 2, "legendFormat": "conflicts", @@ -1015,32 +923,30 @@ "refId": "B", "resultFormat": "time_series", "select": [ - [ + [ { - "params": [ - "deadlocks" - ], - "type": "field" + "params": ["deadlocks"], + "type": "field" }, { - "params": [], - "type": "mean" + "params": [], + "type": "mean" }, { - "params": [], - "type": "difference" + "params": [], + "type": "difference" } - ] + ] ], "step": 240, "tags": [ - { + { "key": "instance", "operator": "=~", "value": "/^$instance$/" - } + } ] - } + } ], "thresholds": [], "timeFrom": null, @@ -1048,83 +954,82 @@ "timeShift": null, "title": "Conflicts/Deadlocks", "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" }, "type": "graph", "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] }, "yaxes": [ - { + { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true - }, - { + }, + { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true - } + } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false, + "alignLevel": null } - }, - { + }, + { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { - "defaults": { + "defaults": { "custom": {} - }, - "overrides": [] + }, + "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 14 + "h": 7, + "w": 12, + "x": 0, + "y": 14 }, "hiddenSeries": false, "id": 12, "isNew": true, "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { - "alertThreshold": true + "alertThreshold": true }, "percentage": true, "pluginVersion": "7.2.1", @@ -1136,14 +1041,14 @@ "stack": false, "steppedLine": false, "targets": [ - { + { "expr": "sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=~\"$instance\"}) / (sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=~\"$instance\"}) + sum(pg_stat_database_blks_read{datname=~\"$db\",instance=~\"$instance\"}))", "format": "time_series", "intervalFactor": 2, "legendFormat": "cache hit rate", "refId": "A", "step": 240 - } + } ], "thresholds": [], "timeFrom": null, @@ -1151,83 +1056,82 @@ "timeShift": null, "title": "Cache hit ratio", "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" }, "type": "graph", "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] }, "yaxes": [ - { + { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true - }, - { + }, + { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true - } + } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false, + "alignLevel": null } - }, - { + }, + { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { - "defaults": { + "defaults": { "custom": {} - }, - "overrides": [] + }, + "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 14 + "h": 7, + "w": 12, + "x": 12, + "y": 14 }, "hiddenSeries": false, "id": 13, "isNew": true, "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { - "alertThreshold": true + "alertThreshold": true }, "percentage": false, "pluginVersion": "7.2.1", @@ -1239,14 +1143,14 @@ "stack": false, "steppedLine": false, "targets": [ - { + { "expr": "pg_stat_database_numbackends{datname=~\"$db\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{__name__}}", "refId": "A", "step": 240 - } + } ], "thresholds": [], "timeFrom": null, @@ -1254,161 +1158,156 @@ "timeShift": null, "title": "Number of active connections", "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" }, "type": "graph", "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] }, "yaxes": [ - { + { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true - }, - { + }, + { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true - } + } ], "yaxis": { - "align": false, - "alignLevel": null - } + "align": false, + "alignLevel": null } + } ], "refresh": false, "schemaVersion": 26, "style": "dark", - "tags": [ - "postgres" - ], + "tags": ["postgres"], "templating": { - "list": [ + "list": [ { - "allValue": ".*", - "current": { + "allValue": ".*", + "current": { "selected": false, "text": "All", "value": "$__all" - }, - "datasource": "$datasource", - "definition": "", - "hide": 0, - "includeAll": true, - "label": null, - "multi": false, - "name": "instance", - "options": [], - "query": "label_values(up{job=~\"postgres.*\"},instance)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(up{job=~\"postgres.*\"},instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false }, { - "allValue": ".*", - "current": { + "allValue": ".*", + "current": { "selected": false, "text": "All", "value": "$__all" - }, - "datasource": "$datasource", - "definition": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", - "hide": 0, - "includeAll": true, - "label": "db", - "multi": false, - "name": "db", - "options": [], - "query": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + }, + "definition": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", + "hide": 0, + "includeAll": true, + "label": "db", + "multi": false, + "name": "db", + "options": [], + "query": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false }, { - "current": { + "current": { "selected": false, "text": "Postgres Overview", "value": "Postgres Overview" - }, - "hide": 0, - "includeAll": false, - "label": "datasource", - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" }, { - "allValue": null, - "current": { + "allValue": null, + "current": { "selected": true, "text": "postgres", "value": "postgres" - }, - "datasource": "$datasource", - "definition": "label_values(pg_up, job)", - "hide": 0, - "includeAll": false, - "label": "job", - "multi": false, - "name": "job", - "options": [ + }, + "definition": "label_values(pg_up, job)", + "hide": 0, + "includeAll": false, + "label": "job", + "multi": false, + "name": "job", + "options": [ { - "selected": true, - "text": "postgres", - "value": "postgres" + "selected": true, + "text": "postgres", + "value": "postgres" } - ], - "query": "label_values(pg_up, job)", - "refresh": 0, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + ], + "query": "label_values(pg_up, job)", + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false } - ] + ] }, "time": { - "from": "now-1h", - "to": "now" + "from": "now-1h", + "to": "now" }, "timepicker": { - "refresh_intervals": [ + "refresh_intervals": [ "5s", "10s", "30s", @@ -1419,22 +1318,13 @@ "1h", "2h", "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] + ], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] }, "timezone": "browser", "title": "PostgreSQL Exporter Quickstart and Dashboard", "uid": "wGgaPlciz", "version": 5 - } -} \ No newline at end of file + }, + "overwrite": true +} diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_registry.json b/rootfs/usr/share/grafana/api/dashboards/drycc_registry.json index 5a8105c..5a5cf38 100644 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_registry.json +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_registry.json @@ -37,12 +37,7 @@ "cacheTimeout": null, "colorBackground": false, "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_DS_PROMETHEUS}", + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], "description": "start time of the process", "format": "dateTimeFromNow", "gauge": { @@ -119,12 +114,7 @@ "cacheTimeout": null, "colorBackground": true, "colorValue": false, - "colors": [ - "#967302", - "#967302", - "#967302" - ], - "datasource": "${DS_DS_PROMETHEUS}", + "colors": ["#967302", "#967302", "#967302"], "description": "The number of cache request received", "format": "short", "gauge": { @@ -201,12 +191,7 @@ "cacheTimeout": null, "colorBackground": true, "colorValue": false, - "colors": [ - "#1f78c1", - "#1f78c1", - "#1f78c1" - ], - "datasource": "${DS_DS_PROMETHEUS}", + "colors": ["#1f78c1", "#1f78c1", "#1f78c1"], "description": "The number of cache request received (hits)", "format": "short", "gauge": { @@ -283,12 +268,7 @@ "cacheTimeout": null, "colorBackground": true, "colorValue": false, - "colors": [ - "#890f02", - "#890f02", - "#890f02" - ], - "datasource": "${DS_DS_PROMETHEUS}", + "colors": ["#890f02", "#890f02", "#890f02"], "description": "The number of cache request received (miss)", "format": "short", "gauge": { @@ -365,12 +345,7 @@ "cacheTimeout": null, "colorBackground": false, "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_DS_PROMETHEUS}", + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], "description": "Process Resident Memory Usage", "format": "decbytes", "gauge": { @@ -448,7 +423,6 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_DS_PROMETHEUS}", "description": "Average user and system CPU time spent in seconds.", "fill": 1, "gridPos": { @@ -534,7 +508,6 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_DS_PROMETHEUS}", "description": "Virtual and Resident memory size in bytes, averages over 5 min interval", "fill": 1, "gridPos": { @@ -628,7 +601,6 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_DS_PROMETHEUS}", "description": "Number of open file descriptors", "fill": 1, "gridPos": { @@ -728,7 +700,6 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_DS_PROMETHEUS}", "description": "The HTTP requests", "fill": 1, "gridPos": { @@ -814,7 +785,6 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_DS_PROMETHEUS}", "description": "The in-flight HTTP requests", "fill": 1, "gridPos": { @@ -900,7 +870,6 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_DS_PROMETHEUS}", "description": "The rate of registry cache requests", "fill": 1, "gridPos": { @@ -986,7 +955,6 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_DS_PROMETHEUS}", "fill": 1, "gridPos": { "h": 7, @@ -1083,7 +1051,6 @@ "mode": "spectrum" }, "dataFormat": "timeseries", - "datasource": "${DS_DS_PROMETHEUS}", "gridPos": { "h": 7, "w": 8, @@ -1142,7 +1109,6 @@ "mode": "spectrum" }, "dataFormat": "timeseries", - "datasource": "${DS_DS_PROMETHEUS}", "gridPos": { "h": 7, "w": 8, @@ -1201,7 +1167,6 @@ "mode": "spectrum" }, "dataFormat": "timeseries", - "datasource": "${DS_DS_PROMETHEUS}", "gridPos": { "h": 7, "w": 8, @@ -1260,7 +1225,6 @@ "mode": "spectrum" }, "dataFormat": "timeseries", - "datasource": "${DS_DS_PROMETHEUS}", "gridPos": { "h": 7, "w": 8, @@ -1319,7 +1283,6 @@ "mode": "spectrum" }, "dataFormat": "timeseries", - "datasource": "${DS_DS_PROMETHEUS}", "gridPos": { "h": 7, "w": 8, @@ -1369,17 +1332,12 @@ "refresh": "10s", "schemaVersion": 16, "style": "dark", - "tags": [ - "docker", - "registry", - "internals" - ], + "tags": ["docker", "registry", "internals"], "templating": { "list": [ { "allValue": null, "current": {}, - "datasource": "${DS_DS_PROMETHEUS}", "hide": 0, "includeAll": false, "label": null, @@ -1399,7 +1357,6 @@ { "allValue": null, "current": {}, - "datasource": "${DS_DS_PROMETHEUS}", "hide": 0, "includeAll": false, "label": null, @@ -1419,7 +1376,6 @@ { "allValue": null, "current": {}, - "datasource": "${DS_DS_PROMETHEUS}", "hide": 0, "includeAll": false, "label": null, @@ -1455,21 +1411,12 @@ "2h", "1d" ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] }, "timezone": "", "title": "Docker Registry", "uid": "CoBSgj8iz", "version": 19 - } -} \ No newline at end of file + }, + "overwrite": true +} diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json b/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json index ed50bcb..c6df255 100644 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json @@ -33,9 +33,7 @@ "icon": "external link", "includeVars": true, "keepTime": true, - "tags": [ - "minio" - ], + "tags": ["minio"], "type": "dashboards" } ], @@ -85,9 +83,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "mean" - ], + "calcs": ["mean"], "fields": "", "values": false }, @@ -158,9 +154,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "last" - ], + "calcs": ["last"], "fields": "", "values": false }, @@ -265,15 +259,11 @@ "displayMode": "table", "placement": "bottom", "showLegend": true, - "values": [ - "percent" - ] + "values": ["percent"] }, "pieType": "donut", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -384,9 +374,7 @@ "id": "byNames", "options": { "mode": "exclude", - "names": [ - "Usage" - ], + "names": ["Usage"], "prefix": "All except:", "readOnly": true } @@ -474,9 +462,7 @@ "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -648,9 +634,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "last" - ], + "calcs": ["last"], "fields": "", "values": false }, @@ -807,9 +791,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -869,9 +851,7 @@ "minVizWidth": 75, "orientation": "auto", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -962,9 +942,7 @@ "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -1218,9 +1196,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "mean" - ], + "calcs": ["mean"], "fields": "", "values": false }, @@ -1299,9 +1275,7 @@ "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -1364,9 +1338,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "last" - ], + "calcs": ["last"], "fields": "", "values": false }, @@ -1430,9 +1402,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "last" - ], + "calcs": ["last"], "fields": "", "values": false }, @@ -1764,9 +1734,7 @@ "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -1885,9 +1853,7 @@ "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -2044,9 +2010,7 @@ "minVizWidth": 75, "orientation": "auto", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -2681,9 +2645,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "mean" - ], + "calcs": ["mean"], "fields": "", "values": false }, @@ -3248,9 +3210,7 @@ ], "refresh": "", "schemaVersion": 39, - "tags": [ - "minio" - ], + "tags": ["minio"], "templating": { "list": [ { @@ -3289,17 +3249,7 @@ "2h", "1d" ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] }, "timezone": "", "title": "MinIO Dashboard", @@ -3307,5 +3257,5 @@ "version": 54, "weekStart": "" }, - "overwrite": false + "overwrite": true } diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_valkey.json b/rootfs/usr/share/grafana/api/dashboards/drycc_valkey.json index fbca559..a712c13 100644 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_valkey.json +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_valkey.json @@ -13,11 +13,11 @@ } ] }, - "description": "A quickstart to setup the Prometheus Redis Exporter with preconfigured dashboards, alerting rules, and recording rules.", + "description": "Redis Dashboard for Prometheus Redis Exporter 1.x", "editable": true, - "gnetId": 14091, + "gnetId": 763, "graphTooltip": 1, - "id": 24, + "id": null, "iteration": 1602758020790, "links": [], "panels": [ @@ -90,7 +90,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(irate(redis_commands_total{instance=~\"$instance\"} [1m])) by (cmd)", + "expr": "avg(irate(redis_commands_total{instance=~\"$instance\"} [$__rate_interval])) by (cmd)", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -197,7 +197,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(irate(redis_commands_duration_seconds_total{instance=~\"$instance\"}[1m])) by (cmd)\n /\navg(irate(redis_commands_total{instance=~\"$instance\"}[1m])) by (cmd)\n", + "expr": "avg(irate(redis_commands_duration_seconds_total{instance=~\"$instance\"}[$__rate_interval])) by (cmd)\n /\navg(irate(redis_commands_total{instance=~\"$instance\"}[$__rate_interval])) by (cmd)\n", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -314,7 +314,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(irate(redis_keyspace_hits_total{instance=~\"$instance\"}[1m]) / (irate(redis_keyspace_misses_total{instance=~\"$instance\"}[1m]) + irate(redis_keyspace_hits_total{instance=~\"$instance\"}[1m]))) by (instance)", + "expr": "avg(irate(redis_keyspace_hits_total{instance=~\"$instance\"}[$__rate_interval]) / (irate(redis_keyspace_misses_total{instance=~\"$instance\"}[$__rate_interval]) + irate(redis_keyspace_hits_total{instance=~\"$instance\"}[$__rate_interval]))) by (instance)", "format": "time_series", "hide": false, "interval": "", @@ -697,7 +697,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(redis_evicted_keys_total{instance=~\"$instance\"}[1m])", + "expr": "irate(redis_evicted_keys_total{instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1289,10 +1289,7 @@ "refresh": false, "schemaVersion": 25, "style": "dark", - "tags": [ - "prometheus", - "redis" - ], + "tags": ["prometheus", "redis"], "templating": { "list": [ { @@ -1301,10 +1298,7 @@ "selected": true, "tags": [], "text": "redisdb-7d6b98cd98-kjt5x + redisdb-7d6b98cd98-pbkg6", - "value": [ - "redisdb-7d6b98cd98-kjt5x", - "redisdb-7d6b98cd98-pbkg6" - ] + "value": ["redisdb-7d6b98cd98-kjt5x", "redisdb-7d6b98cd98-pbkg6"] }, "datasource": "${datasource}", "definition": "label_values(redis_up, instance)", @@ -1333,7 +1327,7 @@ }, "hide": 0, "includeAll": false, - "label": null, + "label": "Data Source", "multi": false, "name": "datasource", "options": [], @@ -1362,21 +1356,12 @@ "2h", "1d" ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] }, "timezone": "browser", - "title": "Redis Exporter Quickstart and Dashboard", + "title": "Redis Dashboard for Prometheus Redis Exporter 1.x", "uid": "bRd48yKMdd", "version": 5 - } -} \ No newline at end of file + }, + "overwrite": true +} diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_cluster.json b/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_cluster.json index e76ae97..a6e3d6e 100644 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_cluster.json +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_cluster.json @@ -144,9 +144,7 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -215,9 +213,7 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -288,9 +284,7 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -360,9 +354,7 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -432,9 +424,7 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -504,9 +494,7 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -576,9 +564,7 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -647,9 +633,7 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -751,9 +735,7 @@ "footer": { "countRows": false, "fields": "", - "reducer": [ - "sum" - ], + "reducer": ["sum"], "show": false }, "showHeader": true, @@ -856,9 +838,7 @@ "id": 62, "options": { "legend": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true, @@ -978,11 +958,7 @@ "id": 2, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -1087,11 +1063,7 @@ "id": 6, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -1196,11 +1168,7 @@ "id": 12, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -1303,11 +1271,7 @@ "id": 8, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -1412,11 +1376,7 @@ "id": 52, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -1527,11 +1487,7 @@ "id": 104, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -1655,11 +1611,7 @@ "id": 66, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -1770,11 +1722,7 @@ "id": 138, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -1884,11 +1832,7 @@ "id": 64, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -2007,11 +1951,7 @@ "id": 122, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -2151,11 +2091,7 @@ "id": 117, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -2274,11 +2210,7 @@ "id": 204, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -2399,11 +2331,7 @@ "id": 68, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -2504,11 +2432,7 @@ "id": 119, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -2610,11 +2534,7 @@ "id": 70, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -2715,11 +2635,7 @@ "id": 120, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -2822,11 +2738,7 @@ "id": 210, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -2928,11 +2840,7 @@ "id": 213, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -3093,11 +3001,7 @@ "id": 102, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -3212,11 +3116,7 @@ "id": 108, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -3318,11 +3218,7 @@ "id": 170, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -3428,11 +3324,7 @@ "id": 107, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -3532,11 +3424,7 @@ "id": 144, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -3640,11 +3528,7 @@ "id": 58, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -3759,11 +3643,7 @@ ], "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -3869,11 +3749,7 @@ "links": [], "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -3976,11 +3852,7 @@ "id": 135, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -4100,9 +3972,7 @@ "id": 217, "options": { "legend": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -4204,9 +4074,7 @@ "footer": { "countRows": false, "fields": "", - "reducer": [ - "sum" - ], + "reducer": ["sum"], "show": false }, "showHeader": true, @@ -4241,9 +4109,7 @@ "options": { "fields": { "instance": { - "aggregations": [ - "uniqueValues" - ], + "aggregations": ["uniqueValues"], "operation": "aggregate" }, "job": { @@ -4338,9 +4204,7 @@ "id": 214, "options": { "legend": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -4454,11 +4318,7 @@ "id": 76, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4569,11 +4429,7 @@ "id": 86, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4693,11 +4549,7 @@ "id": 80, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4797,11 +4649,7 @@ "id": 78, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4912,11 +4760,7 @@ "id": 82, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -5019,11 +4863,7 @@ "id": 74, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -5137,11 +4977,7 @@ "id": 100, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -5252,11 +5088,7 @@ "id": 113, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "min" - ], + "calcs": ["mean", "lastNotNull", "min"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -5368,11 +5200,7 @@ "id": 151, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -5517,11 +5345,7 @@ "id": 167, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -5681,11 +5505,7 @@ ], "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -5818,11 +5638,7 @@ "id": 133, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -5942,11 +5758,7 @@ "id": 54, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -6050,11 +5862,7 @@ "id": 55, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -6163,11 +5971,7 @@ "id": 20, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -6300,11 +6104,7 @@ "id": 22, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -6427,11 +6227,7 @@ "id": 202, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -6567,11 +6363,7 @@ "id": 14, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -6703,10 +6495,7 @@ "id": 206, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], + "calcs": ["mean", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -7045,11 +6834,7 @@ "id": 92, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -7170,11 +6955,7 @@ "id": 95, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -7291,11 +7072,7 @@ "id": 163, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -7434,11 +7211,7 @@ "id": 165, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -7573,11 +7346,7 @@ "id": 178, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -7679,11 +7448,7 @@ "id": 180, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -7785,11 +7550,7 @@ "id": 179, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -7891,11 +7652,7 @@ "id": 181, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -8008,10 +7765,7 @@ "id": 93, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], + "calcs": ["mean", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -8137,10 +7891,7 @@ "id": 207, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], + "calcs": ["mean", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -8268,11 +8019,7 @@ "id": 97, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -8393,11 +8140,7 @@ "id": 99, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -8516,11 +8259,7 @@ "id": 185, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -8659,11 +8398,7 @@ "id": 187, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -8798,10 +8533,7 @@ "id": 139, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], + "calcs": ["mean", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -8904,10 +8636,7 @@ "id": 114, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], + "calcs": ["mean", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -9018,10 +8747,7 @@ "id": 208, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], + "calcs": ["mean", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -9147,10 +8873,7 @@ "id": 209, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], + "calcs": ["mean", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -9265,11 +8988,7 @@ "id": 88, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -9406,11 +9125,7 @@ "id": 189, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -9506,11 +9221,7 @@ "id": 190, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -9606,11 +9317,7 @@ "id": 192, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -9712,11 +9419,7 @@ "id": 196, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "min" - ], + "calcs": ["mean", "lastNotNull", "min"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -9813,11 +9516,7 @@ "id": 200, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -9913,11 +9612,7 @@ "id": 201, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -10031,11 +9726,7 @@ "id": 203, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -10227,5 +9918,6 @@ "version": 1, "weekStart": "", "gnetId": 11176 - } -} \ No newline at end of file + }, + "overwrite": true +} diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_vmagent.json b/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_vmagent.json index 3c473aa..39f8d22 100644 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_vmagent.json +++ b/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_vmagent.json @@ -173,9 +173,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "last" - ], + "calcs": ["last"], "fields": "", "values": false }, @@ -242,9 +240,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "last" - ], + "calcs": ["last"], "fields": "", "values": false }, @@ -306,9 +302,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "last" - ], + "calcs": ["last"], "fields": "", "values": false }, @@ -400,9 +394,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "last" - ], + "calcs": ["last"], "fields": "", "values": false }, @@ -489,9 +481,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "last" - ], + "calcs": ["last"], "fields": "", "values": false }, @@ -555,9 +545,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "last" - ], + "calcs": ["last"], "fields": "", "values": false }, @@ -654,9 +642,7 @@ "footer": { "countRows": false, "fields": "", - "reducer": [ - "sum" - ], + "reducer": ["sum"], "show": false }, "showHeader": true @@ -751,9 +737,7 @@ "id": 13, "options": { "legend": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true, @@ -899,11 +883,7 @@ "id": 5, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -1030,11 +1010,7 @@ ], "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -1137,11 +1113,7 @@ "id": 107, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -1254,11 +1226,7 @@ ], "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -1357,11 +1325,7 @@ "id": 15, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -1467,11 +1431,7 @@ ], "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -1639,11 +1599,7 @@ "id": 109, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -1754,11 +1710,7 @@ "id": 111, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -1879,11 +1831,7 @@ "id": 81, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -2015,11 +1963,7 @@ "id": 7, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -2134,11 +2078,7 @@ "id": 83, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -2240,11 +2180,7 @@ "id": 39, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -2347,11 +2283,7 @@ "id": 135, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -2454,11 +2386,7 @@ "id": 149, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -2560,11 +2488,7 @@ "id": 41, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -2690,11 +2614,7 @@ "id": 92, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -2795,11 +2715,7 @@ "id": 95, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -2903,11 +2819,7 @@ "id": 98, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -3011,11 +2923,7 @@ "id": 99, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -3118,11 +3026,7 @@ "id": 79, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -3232,11 +3136,7 @@ ], "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -3338,11 +3238,7 @@ "id": 127, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -3443,11 +3339,7 @@ "id": 50, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -3547,9 +3439,7 @@ "footer": { "countRows": false, "fields": "", - "reducer": [ - "sum" - ], + "reducer": ["sum"], "show": false }, "showHeader": true, @@ -3677,9 +3567,7 @@ "id": 150, "options": { "legend": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -3806,11 +3694,7 @@ "id": 48, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -3912,11 +3796,7 @@ "id": 76, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4016,11 +3896,7 @@ "id": 132, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -4122,11 +3998,7 @@ "id": 133, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4227,11 +4099,7 @@ "id": 20, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4331,11 +4199,7 @@ "id": 126, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4434,11 +4298,7 @@ "id": 46, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4537,11 +4397,7 @@ "id": 148, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4640,11 +4496,7 @@ "id": 31, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4819,11 +4671,7 @@ "id": 73, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -4936,11 +4784,7 @@ "id": 131, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -5040,11 +4884,7 @@ "id": 130, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -5157,11 +4997,7 @@ "id": 77, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -5291,11 +5127,7 @@ "id": 146, "options": { "legend": { - "calcs": [ - "min", - "lastNotNull", - "max" - ], + "calcs": ["min", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -5489,11 +5321,7 @@ "id": 147, "options": { "legend": { - "calcs": [ - "min", - "lastNotNull", - "max" - ], + "calcs": ["min", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -6044,11 +5872,7 @@ "id": 60, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -6148,11 +5972,7 @@ "id": 66, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -6252,11 +6072,7 @@ "id": 61, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -6356,11 +6172,7 @@ "id": 65, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -6558,11 +6370,7 @@ "id": 84, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -6805,11 +6613,7 @@ "id": 119, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -6909,11 +6713,7 @@ "id": 117, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -7021,11 +6821,7 @@ ], "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -7137,11 +6933,7 @@ "id": 123, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -7267,11 +7059,7 @@ "id": 121, "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], + "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true, @@ -7326,10 +7114,7 @@ ], "refresh": "", "schemaVersion": 39, - "tags": [ - "vmagent", - "victoriametrics" - ], + "tags": ["vmagent", "victoriametrics"], "templating": { "list": [ { @@ -7462,5 +7247,6 @@ "version": 1, "weekStart": "", "gnetId": 12683 - } -} \ No newline at end of file + }, + "overwrite": true +} diff --git a/rootfs/usr/share/grafana/api/dashboards/kubernetes.json b/rootfs/usr/share/grafana/api/dashboards/kubernetes.json new file mode 100644 index 0000000..8815e47 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/kubernetes.json @@ -0,0 +1,5929 @@ +{ + "dashboard": { + "annotations": { + "list": [ + { + "$$hashKey": "object:247", + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "【English version】2025.01.25 Updates，kubernetesComprehensive display of resources！Includes K8S Overall Resource Overview、Microservices Resource Details、Pod Resource Details andK8SNetwork Bandwidth，Optimization metrics。https://grafana.com/orgs/starsliao/dashboards", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "icon": "bolt", + "tags": [], + "targetBlank": true, + "title": "Update", + "tooltip": "See more dashboards", + "type": "link", + "url": "https://grafana.com/orgs/starsliao/dashboards" + }, + { + "$$hashKey": "object:831", + "icon": "question", + "tags": ["node_exporter"], + "targetBlank": true, + "title": "GitHub", + "tooltip": "View more dashboards", + "type": "link", + "url": "https://github.com/starsliao" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 54, + "panels": [], + "title": "Node Resource Overview：Selected Nodes:【$Node】", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 44, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Memory Utilization", + "refId": "C" + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Memory Request Rate", + "refId": "A" + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Memory Limit Rate", + "refId": "B", + "step": 10 + } + ], + "title": "Node Memory Ratio", + "type": "bargauge" + }, + { + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 45, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m])) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "CPU Utilization Rate", + "refId": "C" + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "CPU Request Rate", + "refId": "A" + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "CPU Limit Rate", + "refId": "B" + } + ], + "title": "Node CPU Ratio", + "type": "bargauge" + }, + { + "description": "Number of cluster nodes，Nodes POD Number of，NodesPODUpper Limit", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 1000 + }, + { + "color": "red", + "value": 2000 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 8, + "y": 1 + }, + "id": 74, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "count(kube_node_info{origin_prometheus=~\"$origin_prometheus\"})", + "instant": true, + "interval": "", + "legendFormat": "Number of nodes", + "refId": "A" + }, + { + "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",created_by_kind!~\"|Job\",node=~\"^$Node$\"})", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Pod Number of nodes", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"pods\", unit=\"integer\",node=~\"^$Node$\"})", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Upper limit Pod", + "refId": "C" + } + ], + "title": "Nodes with Pod", + "type": "bargauge" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "color-text" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Space" + }, + "properties": [ + { + "id": "custom.width", + "value": 59 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pod" + }, + "properties": [ + { + "id": "custom.width", + "value": 21 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SVC" + }, + "properties": [ + { + "id": "custom.width", + "value": 7 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Microservices" + }, + "properties": [ + { + "id": "custom.width", + "value": 4 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Configuration" + }, + "properties": [ + { + "id": "custom.width", + "value": 16 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Passwords" + }, + "properties": [ + { + "id": "custom.width", + "value": 33 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 11, + "y": 1 + }, + "id": 51, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": true + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Microservices" + } + ] + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\"}) by (namespace)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "count(kube_service_info{origin_prometheus=~\"$origin_prometheus\"}) by(namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "count by (namespace)({__name__=~\"kube_deployment_metadata_generation|kube_daemonset_metadata_generation|kube_statefulset_metadata_generation\",origin_prometheus=~\"$origin_prometheus\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "refId": "D" + }, + { + "editorMode": "code", + "expr": "count(kube_configmap_info{origin_prometheus=~\"$origin_prometheus\"}) by(namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "configmap", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "count(kube_secret_info{origin_prometheus=~\"$origin_prometheus\"}) by(namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "secret", + "refId": "E" + } + ], + "title": "Namespace Resource Statistics", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "namespace" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true + }, + "includeByName": {}, + "indexByName": { + "Time 1": 2, + "Time 2": 4, + "Time 3": 6, + "Value #A": 3, + "Value #C": 5, + "Value #D": 1, + "namespace": 0 + }, + "renameByName": { + "Time 1": "", + "Time 2": "", + "Value #A": "Pod", + "Value #B": "Configuration", + "Value #C": "SVC", + "Value #D": "Microservices", + "Value #E": "Passwords", + "namespace": "Spaces" + } + } + } + ], + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 32, + "options": { + "legend": { + "calcs": ["max"], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 200 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum (irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m]))*8", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Receive", + "metric": "network", + "range": true, + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "sum (irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m]))*8", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Send", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "title": "$NameSpace：Network Overview（Associable nodes and namespaces）", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "max": 2000000000000, + "min": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 100000000000 + }, + { + "color": "red", + "value": 2000000000000 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 71, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "exemplar": true, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Total Memory", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Usage", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Requests", + "refId": "D" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Limits", + "refId": "B" + } + ], + "title": "Node Memory Information", + "type": "bargauge" + }, + { + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "max": 500, + "min": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 500 + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 72, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "exemplar": true, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Total Cores", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",id=\"/\",node=~\"^$Node$\"}[2m]))", + "instant": true, + "interval": "", + "legendFormat": "Usage", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Requests", + "refId": "D" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Limit", + "refId": "B" + } + ], + "title": "Node CPU Number of cores", + "type": "bargauge" + }, + { + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "max": 8000000000000, + "min": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 5000000000000 + }, + { + "color": "red", + "value": 10000000000000 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Utilization" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "max" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "thresholds", + "value": { + "mode": "percentage", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 90 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "decimals", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 8, + "y": 5 + }, + "id": 73, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"}) / sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Utilization Rate", + "refId": "C" + }, + { + "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Usage", + "refId": "A" + }, + { + "expr": "sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Total", + "refId": "B" + } + ], + "title": "Node Storage Information", + "type": "bargauge" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Anomaly.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 88, + "maxPerRow": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "repeat": "origin_prometheus", + "repeatDirection": "v", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "count({__name__=~\"kube_deployment_metadata_generation|kube_daemonset_metadata_generation|kube_statefulset_metadata_generation\",origin_prometheus=~\"$origin_prometheus\"})", + "hide": false, + "instant": true, + "legendFormat": "Workload", + "range": false, + "refId": "F" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\"})", + "hide": false, + "instant": true, + "legendFormat": "Total Pod", + "range": false, + "refId": "E" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "count by(key,origin_prometheus)(kube_node_spec_taint{origin_prometheus=~\"$origin_prometheus\",key=~\"node.kubernetes.io.*\"})", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": "{{key}}", + "range": false, + "refId": "D" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "count by(origin_prometheus)(kube_node_info{origin_prometheus=~\"$origin_prometheus\"})", + "hide": false, + "instant": true, + "legendFormat": "Total Nodes", + "range": false, + "refId": "C" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "count by(origin_prometheus)(kube_node_info{origin_prometheus=~\"$origin_prometheus\"}) - count by(origin_prometheus)(kube_node_spec_taint{origin_prometheus=~\"$origin_prometheus\",key!~\"node.kubernetes.io.*\"})", + "hide": false, + "instant": true, + "legendFormat": "Normal Node", + "range": false, + "refId": "B" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "count by(key,origin_prometheus)(kube_node_spec_taint{origin_prometheus=~\"$origin_prometheus\",key!~\"node.kubernetes.io.*\"})", + "hide": false, + "instant": true, + "legendFormat": "{{key}}", + "range": false, + "refId": "A" + } + ], + "title": "", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "(node.kubernetes.io/)(.*)", + "renamePattern": "Abnormal:$2" + } + } + ], + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total Memory" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 11 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", + "instant": false, + "interval": "", + "legendFormat": "Total Memory", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})", + "instant": false, + "interval": "", + "legendFormat": "Usage", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", + "hide": true, + "instant": false, + "interval": "", + "legendFormat": "Requests", + "refId": "D" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", + "hide": true, + "instant": false, + "interval": "", + "legendFormat": "Limit", + "refId": "B" + } + ], + "title": "Memory Usage【$Node】", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total Cores" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 11 + }, + "id": 80, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", + "instant": false, + "interval": "", + "legendFormat": "Total Cores", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",id=\"/\",node=~\"^$Node$\"}[2m]))", + "instant": false, + "interval": "", + "legendFormat": "Usage", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", + "hide": true, + "instant": false, + "interval": "", + "legendFormat": "Requests", + "refId": "D" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", + "hide": true, + "instant": true, + "interval": "", + "legendFormat": "Limit", + "refId": "B" + } + ], + "title": "CPU Used Cores【$Node】", + "type": "timeseries" + }, + { + "description": "Number of Cluster Nodes，Nodes POD Number of，Nodes POD Upper Limit", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Upper Limit Pod" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Number of nodes" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.drawStyle", + "value": "points" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.pointSize", + "value": 3 + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 11 + }, + "id": 81, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "exemplar": true, + "expr": "count(kube_node_info{origin_prometheus=~\"$origin_prometheus\"})", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Number of nodes", + "refId": "A" + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",created_by_kind!~\"|Job\",node=~\"^$Node$\"})", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Pod Number", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"pods\", unit=\"integer\",node=~\"^$Node$\"})", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Upper Limit Pod", + "refId": "C" + } + ], + "title": "Pod Number and nodes【$Node】", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Total number of cores.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 75, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "exemplar": true, + "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m]))by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)*100", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{node}}", + "refId": "I" + } + ], + "title": "$Node：Nodes CPU Breakdown", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 76, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "exemplar": true, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)*100", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{node}}", + "refId": "I" + } + ], + "title": "$Node：Node Memory Breakdown", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "←Inflow/Outflow→", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Inflow.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 78, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "exemplar": true, + "expr": "sum (irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\"}[2m]))by (node) *8", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Inflow:{{node}}", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "exemplar": true, + "expr": "sum (irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\"}[2m]))by (node) *8", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Outflows:{{node}}", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "title": "$Node：Node Network Overview", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "CPU Limitations" + }, + "properties": [ + { + "id": "custom.width", + "value": 76 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 71 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Limits" + }, + "properties": [ + { + "id": "custom.width", + "value": 74 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Disk Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 74 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*%" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + }, + { + "id": "custom.width", + "value": 85 + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "(Memory Usage|Total Memory|Memory Request|Memory Limit|Disk Usage|Disk Total)" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nodes" + }, + "properties": [ + { + "id": "custom.width", + "value": 96 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Requests" + }, + "properties": [ + { + "id": "custom.width", + "value": 76 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CPU Requests" + }, + "properties": [ + { + "id": "custom.width", + "value": 75 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "(CPUTotal|MemoryTotal|DiskTotal|PodLimit)" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PodCap" + }, + "properties": [ + { + "id": "custom.width", + "value": 66 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "CPU Core Usage$|Memory Usage$|Disk Usage$|Pod Number of" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Total/" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pod Number of" + }, + "properties": [ + { + "id": "custom.width", + "value": 58 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CPUTotal Cores" + }, + "properties": [ + { + "id": "custom.width", + "value": 69 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Memory" + }, + "properties": [ + { + "id": "custom.width", + "value": 75 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Disk" + }, + "properties": [ + { + "id": "custom.width", + "value": 74 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CPU Core Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 74 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Usage%" + }, + "properties": [ + { + "id": "custom.width", + "value": 102 + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 52, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Memory Usage%" + } + ] + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "exemplar": false, + "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",created_by_kind!~\"|Job\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "pod Number", + "refId": "A" + }, + { + "exemplar": false, + "expr": "kube_node_status_condition{origin_prometheus=~\"$origin_prometheus\",status=\"true\",node=~\"^$Node$\"} == 1", + "format": "table", + "hide": true, + "instant": true, + "interval": "", + "legendFormat": "Status", + "refId": "B" + }, + { + "exemplar": false, + "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m])) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "I" + }, + { + "exemplar": false, + "expr": "kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"} - 0", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + }, + { + "exemplar": false, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "J" + }, + { + "exemplar": false, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"}) by (node) - 0", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "G" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "H" + }, + { + "exemplar": false, + "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "K" + }, + { + "exemplar": false, + "expr": "sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "L" + }, + { + "exemplar": false, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Memory Usage%", + "refId": "M" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Memory Requests%", + "refId": "N" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Memory Limit%", + "refId": "O" + }, + { + "exemplar": false, + "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m]))by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "CPU Usage%", + "refId": "P" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "CPU Requests%", + "refId": "Q" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Memory Limit%", + "refId": "R" + }, + { + "exemplar": false, + "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})by (node) / sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Disk Usage%", + "refId": "S" + }, + { + "exemplar": false, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"pods\", unit=\"integer\",node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "PodCaps", + "refId": "T" + } + ], + "title": "$Node：Node Information Detail", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 10": true, + "Time 11": true, + "Time 12": true, + "Time 13": true, + "Time 14": true, + "Time 15": true, + "Time 16": true, + "Time 17": true, + "Time 18": true, + "Time 19": true, + "Time 2": true, + "Time 20": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true, + "Time 8": true, + "Time 9": true, + "Value #B": true, + "Value #E": false, + "Value #F": false, + "__name__": true, + "app_kubernetes_io_name": true, + "app_kubernetes_io_name 1": true, + "app_kubernetes_io_name 2": true, + "app_kubernetes_io_name 3": true, + "app_kubernetes_io_version": true, + "app_kubernetes_io_version 1": true, + "app_kubernetes_io_version 2": true, + "app_kubernetes_io_version 3": true, + "condition": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "instance 3": true, + "job": true, + "job 1": true, + "job 2": true, + "job 3": true, + "k8s_namespace": true, + "k8s_namespace 1": true, + "k8s_namespace 2": true, + "k8s_namespace 3": true, + "k8s_sname": true, + "k8s_sname 1": true, + "k8s_sname 2": true, + "k8s_sname 3": true, + "origin_prometheus": true, + "origin_prometheus 1": true, + "origin_prometheus 2": true, + "origin_prometheus 3": true, + "resource": true, + "status": true, + "unit": true + }, + "includeByName": {}, + "indexByName": { + "Time": 22, + "Value #A": 2, + "Value #C": 6, + "Value #D": 8, + "Value #E": 16, + "Value #F": 17, + "Value #G": 18, + "Value #H": 19, + "Value #I": 7, + "Value #J": 9, + "Value #K": 11, + "Value #L": 10, + "Value #M": 4, + "Value #N": 13, + "Value #O": 15, + "Value #P": 3, + "Value #Q": 12, + "Value #R": 14, + "Value #S": 5, + "Value #T": 1, + "instance": 23, + "job": 24, + "node": 0, + "origin_prometheus": 25, + "resource": 20, + "unit": 21 + }, + "renameByName": { + "Value #A": "Pod Number of", + "Value #C": "CPU Total Cores", + "Value #D": "Total Memory", + "Value #E": "CPU Requests", + "Value #F": "CPU Limit", + "Value #G": "Memory Requests", + "Value #H": "Memory Limit", + "Value #I": "CPU Core Usage", + "Value #J": "Memory Usage", + "Value #K": "Disk Usage", + "Value #L": "Disk Total", + "Value #M": "Memory Usage%", + "Value #N": "Memory Requests%", + "Value #O": "Memory Limit%", + "Value #P": "CPU Usage%", + "Value #Q": "CPU Requests%", + "Value #R": "CPU Limit%", + "Value #S": "Disk Usage%", + "Value #T": "Pod Limit", + "condition": "Status", + "node": "Node" + } + } + }, + { + "id": "filterFieldsByName", + "options": {} + } + ], + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "color-background" + }, + "inspect": false + }, + "decimals": 0, + "links": [], + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0 + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "red", + "index": 1 + } + }, + "type": "special" + } + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Utilization" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.width", + "value": 54 + }, + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "decimals" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "from": 75, + "result": { + "color": "semi-dark-red", + "index": 0 + }, + "to": 110 + }, + "type": "range" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mounts Pod Number of" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.width", + "value": 59 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Namespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 58 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PVC" + }, + "properties": [ + { + "id": "custom.width", + "value": 94 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 57 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "custom.width", + "value": 54 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 33 + }, + "id": 92, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Usage Rate" + } + ] + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"})", + "format": "table", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}:{{ persistentvolumeclaim }}", + "metric": "container_memory_usage:sort_desc", + "range": false, + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "min by (namespace,persistentvolumeclaim) (kubelet_volume_stats_available_bytes{origin_prometheus=~\"$origin_prometheus\"}) + max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "metric": "container_memory_usage:sort_desc", + "range": false, + "refId": "B", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"}) /(min by (namespace,persistentvolumeclaim) (kubelet_volume_stats_available_bytes{origin_prometheus=~\"$origin_prometheus\"}) + max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"}))*100", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}:{{ persistentvolumeclaim }}", + "metric": "container_memory_usage:sort_desc", + "range": false, + "refId": "C", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "count by (namespace,persistentvolumeclaim)(kube_pod_spec_volumes_persistentvolumeclaims_info{origin_prometheus=~\"$origin_prometheus\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "metric": "container_memory_usage:sort_desc", + "range": false, + "refId": "D", + "step": 10 + } + ], + "title": "PVC Storage Usage", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value #A": "Usage", + "Value #B": "Total", + "Value #C": "Usage Rate", + "Value #D": "Mount Pod Number", + "namespace": "Namespaces", + "persistentvolumeclaim": "PVC" + } + } + } + ], + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 6, + "y": 33 + }, + "id": 86, + "options": { + "legend": { + "calcs": ["last", "max"], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container !=\"\",container!=\"POD\"}[2m])) by (namespace)>0.5", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ namespace }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "title": "Namespaces CPU Usage kernel(>0.5)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 33 + }, + "id": 85, + "options": { + "legend": { + "calcs": ["last", "max"], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container !=\"\",container!=\"POD\"}) by (namespace) > 1*1024*1024*1024", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace} {{ pod }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Namespaces WSS Memory Usage (>1G)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 49, + "panels": [], + "title": "Pod Resource Overview：SelectedPod:【$Pod】", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "displayName": "", + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 80 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Namespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 96 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pod Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 207 + }, + { + "id": "custom.align", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cores Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 71 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Reboot" + }, + "properties": [ + { + "id": "custom.width", + "value": 38 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "decimals" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*%" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.width", + "value": 55 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Restrictions" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nodes" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "Cores in use$|WSS$|RSS$|Survival|Inflow|Outflow" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Container Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 57 + }, + { + "id": "custom.align", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Survival" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used cores" + }, + "properties": [ + { + "id": "custom.width", + "value": 62 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CPU Limits" + }, + "properties": [ + { + "id": "custom.width", + "value": 58 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Limit" + }, + "properties": [ + { + "id": "custom.width", + "value": 68 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Requirements" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "WSS$|RSS$|Memory Requirements$|Memory Limit$|Disk.*$" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "WSS" + }, + "properties": [ + { + "id": "custom.width", + "value": 81 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RSS" + }, + "properties": [ + { + "id": "custom.width", + "value": 74 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CPU Requirements" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Disk Limitations" + }, + "properties": [ + { + "id": "custom.width", + "value": 83 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Disk Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 10737418240 + }, + { + "color": "red", + "value": 16106127360 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "WSS%" + }, + "properties": [ + { + "id": "custom.width", + "value": 77 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/Inflow|Streaming Out/" + }, + "properties": [ + { + "id": "unit", + "value": "binbps" + }, + { + "id": "custom.width", + "value": 80 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 47, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Container Name" + } + ] + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod,node,namespace) / (sum(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}/100000) by (container, pod,node,namespace)) ", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "exemplar": false, + "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "CPUKernel Usage", + "refId": "Q" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "wss%", + "refId": "I" + }, + { + "exemplar": false, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "wss", + "refId": "D" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "rss%", + "refId": "L" + }, + { + "exemplar": false, + "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "rss", + "refId": "K" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "J" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "kube_pod_container_status_restarts_total{\n origin_prometheus=~\"$origin_prometheus\",\n pod=~\"$Pod\",\n container=~\"$Container\",\n container!=\"\",\n container!=\"POD\",\n namespace=~\"$NameSpace\"\n}\n* on (pod) group_left(node) (\n kube_pod_info{\n origin_prometheus=~\"$origin_prometheus\",\n pod=~\"$Pod\",\n node=~\"^$Node$\",\n namespace=~\"$NameSpace\"\n }\n unless on (pod) (\n kube_pod_info{\n origin_prometheus=~\"$origin_prometheus\",\n pod=~\"$Pod\",\n node=~\"^$Node$\",\n namespace=~\"$NameSpace\"\n } offset 1m\n )\n)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "H" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "time() - \n(\n max by(pod, namespace) (kube_pod_created{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\"})\n * on(pod, namespace) group_right(container)\n kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container=~\"$Container\"}\n)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "refId": "R" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "S" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(sum(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "refId": "T" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(sum(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "refId": "U" + } + ], + "title": "$Node：Pod Resource Detail(Associable Nodes)", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 10": true, + "Time 11": true, + "Time 12": true, + "Time 13": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true, + "Time 8": true, + "Time 9": true, + "Value #G": false, + "__name__": true, + "app_kubernetes_io_name": true, + "app_kubernetes_io_name 1": true, + "app_kubernetes_io_name 2": true, + "app_kubernetes_io_version": true, + "app_kubernetes_io_version 1": true, + "app_kubernetes_io_version 2": true, + "container 1": true, + "container 10": true, + "container 11": true, + "container 12": true, + "container 2": true, + "container 3": true, + "container 4": true, + "container 5": true, + "container 6": true, + "container 7": true, + "container 8": true, + "container 9": true, + "created_by_kind": true, + "created_by_name": true, + "host_ip": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "job": true, + "job 1": true, + "job 2": true, + "k8s_namespace": true, + "k8s_namespace 1": true, + "k8s_namespace 2": true, + "k8s_sname": true, + "k8s_sname 1": true, + "k8s_sname 2": true, + "namespace": false, + "namespace 1": true, + "namespace 10": true, + "namespace 11": true, + "namespace 12": false, + "namespace 2": true, + "namespace 3": true, + "namespace 4": true, + "namespace 5": true, + "namespace 6": true, + "namespace 7": true, + "namespace 8": true, + "namespace 9": true, + "node 1": true, + "node 10": true, + "node 11": false, + "node 12": true, + "node 2": true, + "node 3": true, + "node 4": true, + "node 5": true, + "node 6": true, + "node 7": true, + "node 8": true, + "node 9": true, + "origin_prometheus": true, + "origin_prometheus 1": true, + "origin_prometheus 2": true, + "phase": true, + "pod_ip": true, + "priority_class": true, + "uid": true + }, + "includeByName": {}, + "indexByName": { + "Time": 21, + "Value #A": 4, + "Value #B": 16, + "Value #C": 7, + "Value #D": 10, + "Value #E": 17, + "Value #F": 9, + "Value #G": 23, + "Value #H": 14, + "Value #I": 5, + "Value #J": 13, + "Value #K": 11, + "Value #L": 6, + "Value #M": 24, + "Value #N": 25, + "Value #O": 26, + "Value #P": 27, + "Value #Q": 8, + "Value #R": 15, + "Value #S": 12, + "container": 2, + "instance": 18, + "ip": 28, + "job": 19, + "namespace": 1, + "node": 0, + "origin_prometheus": 20, + "pod": 3, + "uid": 22 + }, + "renameByName": { + "Value #A": "CPU%", + "Value #B": "CPU Requirements", + "Value #C": "CPU Limitations", + "Value #D": "WSS", + "Value #E": "Memory Requirements", + "Value #F": "Memory Limit", + "Value #H": "Reboot", + "Value #I": "WSS%", + "Value #J": "Disk Usage", + "Value #K": "RSS", + "Value #L": "RSS%", + "Value #M": "Heap Memory", + "Value #N": "maxHeap", + "Value #O": "Non-Heap", + "Value #P": "maxNon-Heap", + "Value #Q": "Using Cores", + "Value #R": "Survival", + "Value #S": "Disk Limit", + "Value #T": "Inflow", + "Value #U": "Streaming Out", + "container": "Container Name", + "instance": "", + "ip": "POD IP", + "namespace": "Namespace", + "namespace 1": "", + "namespace 12": "Namespace", + "node": "Nodes", + "node 1": "", + "node 11": "Node", + "pod": "Pod Name", + "priority_class": "" + } + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "Node", + "Namespace", + "Container Name", + "Pod Name", + "CPU%", + "WSS%", + "RSS%", + "CPU Restrictions", + "Cores Used", + "Memory Limit", + "WSS", + "RSS", + "Disk Limit", + "Disk Usage", + "Reboot", + "CPUDemand", + "Memory Requirements", + "Inflow", + "Outflow", + "Survival" + ] + } + } + } + ], + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 50 + }, + "id": 58, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "max(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod) / (max(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}/100000) by (container, pod)) * 100", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ pod }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "title": "Pod Containers CPU Utilization (Maximum100%Associable Nodes)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 50 + }, + "id": 27, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "max (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod)/ max(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod) * 100", + "interval": "", + "intervalFactor": 1, + "legendFormat": "WSS：{{ pod }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "max (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod)/ max(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod) * 100", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RSS：{{ pod }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "B", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "(cass_jvm_heap{service=~\"$Container\"} * on (pod_ip) group_right(service) kube_pod_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\"}) / (cass_jvm_heap_max{service=~\"$Container\"} * on (pod_ip) group_right(service) kube_pod_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\"}) * 100", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Heap：{{ pod }}", + "metric": "container_memory_usage:sort_desc", + "refId": "C", + "step": 10 + } + ], + "title": "Pod Container Memory Usage (Associatable Nodes)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 50 + }, + "id": 77, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "max(max(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Inflow:{{ pod}}", + "metric": "network", + "range": true, + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "max(max(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Outflow:{{ pod}}", + "metric": "network", + "range": true, + "refId": "B", + "step": 10 + }, + { + "editorMode": "code", + "expr": "sum(sum(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info) by(pod) *8", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "metric": "network", + "range": true, + "refId": "C", + "step": 10 + }, + { + "editorMode": "code", + "expr": "sum(sum(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info) by(pod) *8", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "metric": "network", + "range": true, + "refId": "D", + "step": 10 + } + ], + "title": "Pod Network bandwidth per second (Associable Nodes)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Limit.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 59 + }, + "id": 82, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "max(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod,node,namespace)", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU Usage：{{ pod }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "max(max(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)) by(container)", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Pod CPU Limitations：{{ container}}", + "metric": "container_cpu", + "refId": "B", + "step": 10 + } + ], + "title": "Pod Containers CPU Core Usage", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Limitations/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 59 + }, + "id": 84, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "max (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "WSS：{{ pod }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "max(max(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)) by(container)", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "PodMemory Limits：{{ container}}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "B", + "step": 10 + } + ], + "title": "Pod Containers WSS Memory Usage (Associable Nodes)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 59 + }, + "id": 83, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "max (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RSS：{{ pod }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Pod Containers RSS Memory Usage (Associable Nodes)", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 61, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*%" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge", + "valueDisplayMode": "color" + } + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Memory Usage$|.*Memory Limits$|.*Memory Requirements$|.*Disk Usage$|.*Disk Limits$" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Namespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 92 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Container Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 187 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total CPU Core Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pod" + }, + "properties": [ + { + "id": "custom.width", + "value": 44 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Average CPU Usage%" + }, + "properties": [ + { + "id": "custom.width", + "value": 116 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Average RSS Memory Usage%" + }, + "properties": [ + { + "id": "custom.width", + "value": 141 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Average WSS Memory Usage%" + }, + "properties": [ + { + "id": "custom.width", + "value": 165 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total CPU Limit" + }, + "properties": [ + { + "id": "custom.width", + "value": 86 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Memory Limit" + }, + "properties": [ + { + "id": "custom.width", + "value": 86 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Limit$/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Memory Usage$|.*Core Usage$/" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RSS Memory Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 107 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total WSS Memory Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 113 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Average Disk Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 96 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 10737418240 + }, + { + "color": "red", + "value": 16106127360 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Average Disk Limit" + }, + "properties": [ + { + "id": "custom.width", + "value": 96 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total CPU Demand" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Memory Demand" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 87, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Average WSS Memory Usage%" + } + ] + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container) / (sum(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}/100000) by (container))", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Total Cores Used", + "refId": "L" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "I" + }, + { + "exemplar": false, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Average Memory%(RSS)", + "refId": "H" + }, + { + "exemplar": false, + "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Total Memory Usage (RSS) ", + "refId": "K" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum by(container) (kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "avg(container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "J" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "count(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by(container,namespace)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "G" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "avg(container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "M" + } + ], + "title": "Microservices (Container Name) Resource Statistics", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 10": true, + "Time 11": true, + "Time 12": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true, + "Time 8": true, + "Time 9": true + }, + "includeByName": {}, + "indexByName": { + "Time": 15, + "Value #A": 3, + "Value #B": 13, + "Value #C": 6, + "Value #D": 9, + "Value #E": 14, + "Value #F": 8, + "Value #G": 2, + "Value #H": 5, + "Value #I": 4, + "Value #J": 12, + "Value #K": 10, + "Value #L": 7, + "Value #M": 11, + "container": 1, + "namespace": 0 + }, + "renameByName": { + "Time 1": "", + "Value #A": "Average CPU Usage%", + "Value #B": "Total CPU Demand", + "Value #C": "Total CPU Restrictions", + "Value #D": "Total WSS Memory Usage", + "Value #E": "Total Memory Requirements", + "Value #F": "Total Memory Limit", + "Value #G": "Pod", + "Value #H": "Average RSS Memory Usage%", + "Value #I": "Average WSS Memory Usage%", + "Value #J": "Average Disk Usage", + "Value #K": "Total RSS Memory Use", + "Value #L": "Total CPU Core Usage", + "Value #M": "Average Disk Limit", + "container": "Container Name", + "namespace": "Namespace" + } + } + }, + { + "id": "filterFieldsByName", + "options": {} + } + ], + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 24, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "exemplar": true, + "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container) / (sum(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}/100000) by (container)) * 100", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ container}}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "title": "Microservices (Container Name) Average CPU Usage (Maximum100%)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 12 + }, + "id": 89, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container) * 100", + "interval": "", + "intervalFactor": 1, + "legendFormat": "WSS：{{ container }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + }, + { + "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container) * 100", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RSS：{{ container }}", + "metric": "container_memory_usage:sort_desc", + "refId": "B", + "step": 10 + } + ], + "title": "Microservice (Container Name) Average Memory Utilization", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 16, + "options": { + "legend": { + "calcs": ["mean", "last", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(sum(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(container) *8", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Inflow:{{ container }}", + "metric": "network", + "range": true, + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(sum(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(container) *8", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Outflow:{{ container }}", + "metric": "network", + "range": true, + "refId": "B", + "step": 10 + }, + { + "expr": "sum (rate (container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "-> {{ pod }}", + "metric": "network", + "refId": "C", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "<- {{ pod }}", + "metric": "network", + "refId": "D", + "step": 10 + } + ], + "title": "Microservice (Container Name) Network bandwidth per second (Associable Nodes)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/CPU Limit.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 91, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU Limitations：{{ container}}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container)", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU Core Usage：{{ container}}", + "metric": "container_cpu", + "refId": "B", + "step": 10 + } + ], + "title": "Microservices (Container Name) Overall CPU Cores Used", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Memory Limit.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 90, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "WSS：{{ container }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "expr": "sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Memory Limit：{{ container }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "B", + "step": 10 + }, + { + "editorMode": "code", + "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "RSS：{{ container }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "C", + "step": 10 + } + ], + "title": "Microservices (Container Name) Overall Memory Usage", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 59, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "editorMode": "code", + "expr": "count(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by(container,namespace)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}：{{ container }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Microservice (Container Name) Pod Number", + "type": "timeseries" + } + ], + "title": "Microservices (Container Name) Resource Overview：Selected Microservices:【$Container】", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": ["Prometheus", "Kubernetes"], + "templating": { + "list": [ + { + "current": { + "text": "", + "value": "" + }, + "definition": "label_values(kube_node_info,origin_prometheus)", + "includeAll": false, + "label": "K8S", + "name": "origin_prometheus", + "options": [], + "query": { + "query": "label_values(kube_node_info,origin_prometheus)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 5, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(kube_node_info{origin_prometheus=~\"$origin_prometheus\"},node)", + "includeAll": true, + "label": "Nodes", + "name": "Node", + "options": [], + "query": { + "query": "label_values(kube_node_info{origin_prometheus=~\"$origin_prometheus\"},node)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 5, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(kube_namespace_created{origin_prometheus=~\"$origin_prometheus\"},namespace)", + "includeAll": true, + "label": "Namespace", + "name": "NameSpace", + "options": [], + "query": { + "query": "label_values(kube_namespace_created{origin_prometheus=~\"$origin_prometheus\"},namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 5, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\"},container)", + "includeAll": true, + "label": "Microservice (Container Name)", + "name": "Container", + "options": [], + "query": { + "query": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\"},container)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 5, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container=~\"$Container\"},pod)", + "includeAll": true, + "label": "Pod", + "name": "Pod", + "options": [], + "query": { + "query": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container=~\"$Container\"},pod)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": ["30s", "1m", "5m", "15m", "30m", "1h"] + }, + "timezone": "", + "title": "K8S Dashboard EN 20250125", + "uid": "15661", + "version": 3 + }, + "overwrite": true +} diff --git a/rootfs/usr/share/grafana/api/dashboards/kubernetes_apiserver.json b/rootfs/usr/share/grafana/api/dashboards/kubernetes_apiserver.json new file mode 100644 index 0000000..5f5ae6f --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/kubernetes_apiserver.json @@ -0,0 +1,642 @@ +{ + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": 12006, + "graphTooltip": 0, + "id": null, + "iteration": 1585691077160, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[5m])) by (verb)", + "legendFormat": "{{verb}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "apiserver request rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "req/s", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(workqueue_queue_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(workqueue_queue_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p50", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "workqueue service time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "service time", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "95th percentile request latency for Kubernetes apiserver component", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\"CONNECT|WATCH\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\"CONNECT|WATCH\"}[5m])) by (le))", + "legendFormat": "p90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\"CONNECT|WATCH\"}[5m])) by (le))", + "legendFormat": "p50 ", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "apiserver request latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": "latency", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(workqueue_work_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.9, sum(rate(workqueue_work_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(workqueue_work_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p50", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "workqueue processing time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "processing time", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 6, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_helper_cache_hit_total[5m]))/sum((rate(etcd_helper_cache_hit_total[5m])+rate(etcd_helper_cache_miss_total[5m])))", + "legendFormat": "ratio", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "etcd helper cache hit ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "etcd request latency (p95)", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p50", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "etcd request latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": "latency", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Kubernetes apiserver", + "uid": "R6abPf9Zz", + "version": 3, + "description": "Latency and cache hit graphs for Kubernetes apiserver. Useful for observing performance of Amazon EKS clusters." + }, + "overwrite": true +} diff --git a/rootfs/usr/share/grafana/api/dashboards/kubernetes_cluster.json b/rootfs/usr/share/grafana/api/dashboards/kubernetes_cluster.json new file mode 100644 index 0000000..c402452 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/kubernetes_cluster.json @@ -0,0 +1,2013 @@ +{ + "dashboard": { + "id": null, + "title": "Kubernetes cluster monitoring (via Prometheus)", + "description": "Monitors Kubernetes cluster using Prometheus. Shows overall cluster CPU / Memory / Filesystem usage as well as individual pod, containers, systemd services statistics. Uses cAdvisor metrics only.", + "tags": ["kubernetes"], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "200px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)", + "thresholdLine": false + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate (container_network_receive_bytes_total{kubernetes_io_hostname=~\"^$Node$\"}[1m]))", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{kubernetes_io_hostname=~\"^$Node$\"}[1m]))", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O pressure", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Network I/O pressure" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "180px", + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) / sum (machine_memory_bytes{kubernetes_io_hostname=~\"^$Node$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Cluster memory usage", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "180px", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) / sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Cluster CPU usage (1m avg)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "180px", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (container_fs_usage_bytes{device=~\"^/dev/.*$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) / sum (container_fs_limit_bytes{device=~\"^/dev/.*$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Cluster filesystem usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "20%", + "prefix": "", + "prefixFontSize": "20%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (machine_memory_bytes{kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m]))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (container_fs_usage_bytes{device=~\"^/dev/.*$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "id": 14, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (container_fs_limit_bytes{device=~\"^/dev/.*$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "showTitle": false, + "title": "Total usage" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": "", + "id": 17, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (pod_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ pod_name }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Pods CPU usage (1m avg)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "showTitle": false, + "title": "Pods CPU usage" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": "", + "id": 23, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{systemd_service_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (systemd_service_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ systemd_service_name }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "System services CPU usage (1m avg)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "System services CPU usage" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": "", + "id": 24, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",name=~\"^k8s_.*\",container_name!=\"POD\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (container_name, pod_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "pod: {{ pod_name }} | {{ container_name }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, name, image)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "container_cpu", + "refId": "B", + "step": 10 + }, + { + "expr": "sum (rate (container_cpu_usage_seconds_total{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, rkt_container_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "container_cpu", + "refId": "C", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Containers CPU usage (1m avg)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Containers CPU usage" + }, + { + "collapse": true, + "editable": true, + "height": "500px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 20, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (id)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ id }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "All processes CPU usage (1m avg)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "showTitle": false, + "title": "All processes CPU usage" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 25, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}) by (pod_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ pod_name }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Pods memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Pods memory usage" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 26, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{systemd_service_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}) by (systemd_service_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ systemd_service_name }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "System services memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "System services memory usage" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 27, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{image!=\"\",name=~\"^k8s_.*\",container_name!=\"POD\",kubernetes_io_hostname=~\"^$Node$\"}) by (container_name, pod_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "pod: {{ pod_name }} | {{ container_name }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + }, + { + "expr": "sum (container_memory_working_set_bytes{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}) by (kubernetes_io_hostname, name, image)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "container_memory_usage:sort_desc", + "refId": "B", + "step": 10 + }, + { + "expr": "sum (container_memory_working_set_bytes{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}) by (kubernetes_io_hostname, rkt_container_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "container_memory_usage:sort_desc", + "refId": "C", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Containers memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Containers memory usage" + }, + { + "collapse": true, + "editable": true, + "height": "500px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 28, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) by (id)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ id }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "All processes memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "All processes memory usage" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 16, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (pod_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> {{ pod_name }}", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (pod_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- {{ pod_name }}", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Pods network I/O (1m avg)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Pods network I/O" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 30, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (container_name, pod_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> pod: {{ pod_name }} | {{ container_name }}", + "metric": "network", + "refId": "B", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (container_name, pod_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- pod: {{ pod_name }} | {{ container_name }}", + "metric": "network", + "refId": "D", + "step": 10 + }, + { + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, name, image)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, name, image)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "network", + "refId": "C", + "step": 10 + }, + { + "expr": "sum (rate (container_network_transmit_bytes_total{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, rkt_container_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "network", + "refId": "E", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, rkt_container_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "network", + "refId": "F", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Containers network I/O (1m avg)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Containers network I/O" + }, + { + "collapse": true, + "editable": true, + "height": "500px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 29, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate (container_network_receive_bytes_total{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (id)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> {{ id }}", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (id)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- {{ id }}", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "All processes network I/O (1m avg)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "All processes network I/O" + } + ], + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "templating": { + "list": [ + { + "allValue": ".*", + "current": {}, + "hide": 0, + "includeAll": true, + "multi": false, + "name": "Node", + "options": [], + "query": "label_values(kubernetes_io_hostname)", + "refresh": 1, + "type": "query" + } + ] + }, + "annotations": { + "list": [] + }, + "refresh": "10s", + "schemaVersion": 12, + "version": 13, + "uid": "315", + "links": [], + "gnetId": 1621 + }, + "overwrite": true +} diff --git a/rootfs/usr/share/grafana/api/dashboards/kubernetes_deployment_metrics.json b/rootfs/usr/share/grafana/api/dashboards/kubernetes_deployment_metrics.json index 8d09965..18e88d4 100644 --- a/rootfs/usr/share/grafana/api/dashboards/kubernetes_deployment_metrics.json +++ b/rootfs/usr/share/grafana/api/dashboards/kubernetes_deployment_metrics.json @@ -84,9 +84,7 @@ "options": { "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -164,9 +162,7 @@ "options": { "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -239,9 +235,7 @@ "options": { "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "mean" - ], + "calcs": ["mean"], "fields": "", "values": false }, @@ -313,9 +307,7 @@ "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "mean" - ], + "calcs": ["mean"], "fields": "", "values": false }, @@ -387,9 +379,7 @@ "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "mean" - ], + "calcs": ["mean"], "fields": "", "values": false }, @@ -458,9 +448,7 @@ "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "mean" - ], + "calcs": ["mean"], "fields": "", "values": false }, @@ -533,9 +521,7 @@ "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "mean" - ], + "calcs": ["mean"], "fields": "", "values": false }, @@ -604,9 +590,7 @@ "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "mean" - ], + "calcs": ["mean"], "fields": "", "values": false }, @@ -677,9 +661,7 @@ "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "mean" - ], + "calcs": ["mean"], "fields": "", "values": false }, @@ -1211,9 +1193,7 @@ "refresh": "10s", "schemaVersion": 37, "style": "dark", - "tags": [ - "kubernetes" - ], + "tags": ["kubernetes"], "templating": { "list": [ { @@ -1243,12 +1223,8 @@ "allValue": ".*", "current": { "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": ["All"], + "value": ["$__all"] }, "datasource": {}, "definition": "label_values({namespace=\"$Namespace\"},deployment)", @@ -1274,12 +1250,8 @@ "allValue": ".*", "current": { "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": ["All"], + "value": ["$__all"] }, "definition": "label_values({namespace=\"$Namespace\"},statefulset)", "hide": 0, @@ -1343,22 +1315,13 @@ "2h", "1d" ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] }, "timezone": "browser", "title": "Kubernetes Deployment metrics", "uid": "wdtm7Mc7z123", "version": 1, "weekStart": "" - } -} \ No newline at end of file + }, + "overwrite": true +} diff --git a/rootfs/usr/share/grafana/api/dashboards/kubernetes_metrics.json b/rootfs/usr/share/grafana/api/dashboards/kubernetes_metrics.json new file mode 100644 index 0000000..8607f45 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/kubernetes_metrics.json @@ -0,0 +1,3393 @@ +{ + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Summary metrics about kube-state-metrics v2 version(https://github.com/kubernetes/kube-state-metrics); Referenced 6417", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": false, + "tags": ["kubernetes-app"], + "title": "Dashboards", + "type": "dashboards" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 58, + "panels": [], + "title": "Cluster", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "#d44a3a", + "value": 90 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 4, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_info{cluster=~\"$cluster\",node=~\"$node\"}) / sum(kube_node_status_allocatable{cluster=~\"$cluster\",resource=\"pods\",node=~\"$node\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Cluster Pod Requested", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "#d44a3a", + "value": 90 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 5, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests{cluster=~\"$cluster\",resource=\"cpu\",node=~\"$node\"})/ sum(kube_node_status_allocatable{node=~\"$node\",cluster=~\"$cluster\",resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Cluster CPU Requested", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "#d44a3a", + "value": 90 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 6, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests{cluster=~\"$cluster\",resource=\"memory\",node=~\"$node\"}) / sum(kube_node_status_allocatable{node=~\"$node\",cluster=~\"$cluster\",resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Cluster Memory Requested", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "pods", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 9, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable{cluster=~\"$cluster\",resource=\"pods\",node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "allocatable", + "refId": "A" + }, + { + "expr": "sum(kube_pod_info{node=~\"$node\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "requested", + "refId": "C" + } + ], + "title": "Cluster Pod Capacity", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "cores", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 10, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_node_status_capacity{node=~\"$node\",cluster=~\"$cluster\",resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "allocatable", + "refId": "A" + }, + { + "expr": "sum(kube_node_status_allocatable{node=~\"$node\",cluster=~\"$cluster\",resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "capacity", + "refId": "B" + }, + { + "expr": "sum(kube_pod_container_resource_requests{cluster=~\"$cluster\",resource=\"cpu\",node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "requested", + "refId": "C" + }, + { + "expr": "sum(kube_pod_container_resource_limits{cluster=~\"$cluster\",resource=\"cpu\",node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "limited", + "refId": "D" + } + ], + "title": "Cluster CPU Capacity", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bits" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 11, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable{node=~\"$node\",cluster=~\"$cluster\",resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "allocatable", + "refId": "A" + }, + { + "expr": "sum(kube_node_status_capacity{node=~\"$node\",cluster=~\"$cluster\",resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "capacity", + "refId": "B" + }, + { + "expr": "sum(kube_pod_container_resource_requests{cluster=~\"$cluster\",resource=\"memory\",node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "requested", + "refId": "C" + }, + { + "expr": "sum(kube_pod_container_resource_limits{cluster=~\"$cluster\",resource=\"memory\",node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "limited", + "refId": "D" + } + ], + "title": "Cluster Mem Capacity", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 22, + "panels": [ + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 20 + }, + "id": 24, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_node_info{cluster=~\"$cluster\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number Of Nodes", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#d44a3a" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 20 + }, + "id": 26, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_node_spec_unschedulable{cluster=~\"$cluster\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Nodes Unavailable", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 78, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_node_status_condition{cluster=~\"$cluster\",condition=\"Ready\",status=\"false\"}==1", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{node}}", + "refId": "A" + } + ], + "title": "Node NotReady", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 51, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_node_status_condition{condition=\"DiskPressure\",cluster=~\"$cluster\",status=\"true\"}==1", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{node}} {{condition}}", + "refId": "A" + }, + { + "expr": "kube_node_status_condition{condition=\"MemoryPressure\",node=~\"$node\",cluster=~\"$cluster\",status=\"true\"}==1", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} {{condition}}", + "refId": "B" + }, + { + "expr": "kube_node_status_condition{condition=~\"PIDPressure\",node=~\"$node\",cluster=~\"$cluster\",status=\"true\"}==1", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} {{condition}}", + "refId": "C" + } + ], + "title": "Node Pressure", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 88, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max", "min"], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_node_info{cluster=~\"$cluster\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "node number", + "type": "timeseries" + } + ], + "title": "Node", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 14, + "panels": [ + { + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "unit", + "value": "time: YYYY-MM-DD HH:mm:ss" + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Metric" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + {} + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 218 + }, + "id": 16, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_deployment_status_replicas{namespace=~\"$namespace\",cluster=~\"$cluster\"}", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ deployment }}", + "refId": "A" + } + ], + "title": "Deployment Replicas - Up To Date", + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 218 + }, + "id": 18, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_deployment_status_replicas{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Deployment Replicas", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 218 + }, + "id": 19, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_deployment_status_replicas_updated{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Deployment Replicas - Updated", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 218 + }, + "id": 20, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_deployment_status_replicas_unavailable{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Deployment Replicas - Unavailable", + "type": "stat" + } + ], + "title": "Deployments", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 71, + "panels": [ + { + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "unit", + "value": "time: YYYY-MM-DD HH:mm:ss" + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Metric" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + {} + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 22 + }, + "id": 75, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_statefulset_status_replicas_ready{namespace=~\"$namespace\",cluster=~\"$cluster\"}", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ statefulset}}", + "refId": "A" + } + ], + "title": "Statefulset Replicas - Up To Date", + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 6, + "y": 22 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_statefulset_status_replicas_ready{namespace=~\"$namespace\",cluster=~\"$cluster\"}/kube_statefulset_status_replicas{namespace=~\"$namespace\",cluster=~\"$cluster\"}*100", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{statefulset}}", + "refId": "A" + } + ], + "title": "Statefulset replicas", + "type": "timeseries" + } + ], + "title": "Statefuleset", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 28, + "panels": [ + { + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#56A64B" + }, + { + "color": "#73BF69" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "custom.align" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 23 + }, + "id": 68, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "changes(kube_pod_container_status_restarts_total{namespace=~\"$namespace\",cluster=~\"$cluster\"}[30m])>1", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Pods restart in 30m", + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 23 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_pod_status_reason{cluster=~\"$cluster\",namespace=~\"$namespace\", reason=\"Evicted\"}==1", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Evicted", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#629e51", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 30, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_status_phase{cluster=~\"$cluster\",namespace=~\"$namespace\", phase=\"Running\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Pods Running", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#629e51", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 185 + }, + "id": 33, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_status_phase{cluster=~\"$cluster\",namespace=~\"$namespace\", phase=\"Succeeded\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Pods Succeeded", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#629e51", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 185 + }, + "id": 32, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_status_phase{cluster=~\"$cluster\",namespace=~\"$namespace\", phase=\"Failed\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Pods Failed", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#629e51", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 188 + }, + "id": 31, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_status_phase{cluster=~\"$cluster\",namespace=~\"$namespace\", phase=\"Pending\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Pods Pending", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#629e51", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 188 + }, + "id": 34, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_status_phase{cluster=~\"$cluster\",namespace=~\"$namespace\", phase=\"Unknown\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Pods Unknown", + "type": "stat" + } + ], + "title": "Pods", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 36, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 250 + }, + "id": 38, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_container_status_running{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Containers Running", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 250 + }, + "id": 39, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_container_status_waiting{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Containers Waiting", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 250 + }, + "id": 40, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_container_status_terminated{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Containers Terminated", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 250 + }, + "id": 41, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(changes(kube_pod_container_status_restarts_total{namespace=~\"$namespace\",cluster=~\"$cluster\"}[30m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Containers Restarts (Last 30 Minutes)", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 0, + "y": 253 + }, + "id": 43, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "CPU Cores Requested by Containers", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 12, + "y": 253 + }, + "id": 42, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~\"$namespace\" ,cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Memory Requested By Containers", + "type": "stat" + } + ], + "title": "Containers", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 80, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 82, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_hpa_status_current_replicas{cluster=~\"$cluster\",namespace=~\"$namespace\"}", + "instant": false, + "interval": "", + "legendFormat": "current_{{hpa}}", + "refId": "A" + }, + { + "expr": "kube_hpa_spec_max_replicas{cluster=~\"$cluster\",namespace=~\"$namespace\"}", + "instant": false, + "interval": "", + "legendFormat": "max_{{hpa}}", + "refId": "B" + }, + { + "expr": "kube_hpa_spec_min_replicas{cluster=~\"$cluster\",namespace=~\"$namespace\"}", + "instant": false, + "interval": "", + "legendFormat": "min_{{hpa}}", + "refId": "C" + } + ], + "title": "hpa", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 108 + }, + "id": 89, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_hpa_status_current_replicas{hpa=~\".*\"})", + "instant": false, + "interval": "", + "legendFormat": "kube_hpa_status_current_replicas", + "refId": "A" + } + ], + "title": "total-hpa-current", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 116 + }, + "id": 90, + "options": {}, + "pluginVersion": "7.1.1", + "targets": [ + { + "expr": "kube_hpa_status_current_replicas{hpa=~\".*\"} == kube_hpa_spec_max_replicas{hpa=~\".*\"}", + "instant": false, + "interval": "", + "legendFormat": "{{hpa}}", + "refId": "A" + } + ], + "title": "current==max", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 116 + }, + "id": 92, + "options": {}, + "pluginVersion": "7.1.1", + "targets": [ + { + "expr": "kube_hpa_status_current_replicas{hpa=~\".*\"} == kube_hpa_spec_min_replicas{hpa=~\".*\"}", + "instant": false, + "interval": "", + "legendFormat": "{{hpa}}", + "refId": "B" + } + ], + "title": "current==min", + "type": "timeseries" + } + ], + "title": "HPA", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 45, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 26 + }, + "id": 47, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_job_status_succeeded{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Jobs Succeeded", + "type": "stat" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 26 + }, + "id": 62, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "(time()-kube_job_status_completion_time{namespace=~\"$namespace\",cluster=~\"$cluster\"})/60/60/24>1", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{job_name}}", + "refId": "A" + } + ], + "title": "Job last success time from now", + "transparent": true, + "type": "table" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 26 + }, + "id": 76, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_job_status_failed{namespace=~\"$namespace\",cluster=~\"$cluster\"}>1", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{job_name}}", + "refId": "A" + } + ], + "title": "Job failed", + "transparent": true, + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 37 + }, + "id": 48, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_job_status_active{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Jobs Active", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 40 + }, + "id": 49, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_job_status_failed{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Jobs Failed", + "type": "stat" + } + ], + "title": "Jobs", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 64, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 27 + }, + "id": 66, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_persistentvolumeclaim_status_phase{phase=\"Bound\"}==1)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{persistentvolumeclaim=}}", + "refId": "A" + } + ], + "title": "pvc is bound", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 27 + }, + "id": 69, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_persistentvolumeclaim_status_phase{phase!=\"Bound\"}==1)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{persistentvolumeclaim=}}", + "refId": "A" + } + ], + "title": "pvc is not bound", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 67, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_persistentvolumeclaim_resource_requests_storage_bytes/1024/1024/1024", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{persistentvolumeclaim}}", + "refId": "A" + } + ], + "title": "pvc", + "type": "table" + } + ], + "title": "PVC", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": ["kubernetes", "kubernetes-app"], + "templating": { + "list": [ + { + "current": { + "text": "", + "value": "" + }, + "definition": "label_values(kube_node_info, cluster)", + "includeAll": false, + "name": "cluster", + "options": [], + "query": "label_values(kube_node_info, cluster)", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(kube_node_info{cluster=~\"$cluster\"}, node)", + "includeAll": true, + "name": "node", + "options": [], + "query": "label_values(kube_node_info{cluster=~\"$cluster\"}, node)", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "", + "includeAll": true, + "name": "namespace", + "options": [], + "query": "label_values(kube_namespace_created{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 2, + "name": "datasource", + "query": "Prometheus", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "browser", + "title": "kube-state-metrics-v2", + "uid": "garysdevil-kube-state-metrics-v2", + "version": 1 + }, + "overwrite": true +} diff --git a/rootfs/usr/share/grafana/grafana.ini.tpl b/rootfs/usr/share/grafana/grafana.ini.tpl index 03f712a..31a258a 100644 --- a/rootfs/usr/share/grafana/grafana.ini.tpl +++ b/rootfs/usr/share/grafana/grafana.ini.tpl @@ -122,9 +122,6 @@ auto_assign_org_role = {{ default "Viewer" .AUTO_ASSIGN_ORG_ROLE }} # Background text for the user field on the login page login_hint = {{ default "email or username" .LOGIN_HINT }} -#################################### Auth #################################### -[auth] -oauth_auto_login = true #################################### Anonymous Auth ########################## [auth.anonymous] @@ -191,6 +188,7 @@ config_file = "/usr/share/grafana/ldap.toml" {{ if .DRYCC_PASSPORT_URL }} name = OAuth enabled = true +auto_login = true client_id = {{ .DRYCC_PASSPORT_KEY }} client_secret = {{ .DRYCC_PASSPORT_SECRET }} scopes = profile,openid diff --git a/rootfs/usr/share/grafana/start-grafana b/rootfs/usr/share/grafana/start-grafana index 038f4de..f9b42b7 100755 --- a/rootfs/usr/share/grafana/start-grafana +++ b/rootfs/usr/share/grafana/start-grafana @@ -60,7 +60,7 @@ fi set -m echo "Starting Grafana in the background" -exec grafana-server -config /usr/share/grafana/grafana.ini -homepath /opt/drycc/grafana & +exec grafana server -config /usr/share/grafana/grafana.ini -homepath /opt/drycc/grafana & echo "Waiting for Grafana to come up..." until curl -q --fail --output /dev/null --silent "http://${GRAFANA_USER}:${GRAFANA_PASSWD}@localhost:${GRAFANA_PORT}/api/org"; do printf "." From edaab1ce38bbbc95e4f2702767fd2143c5e2b470 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 23 May 2025 16:41:30 +0800 Subject: [PATCH 23/50] feat(grafana): add multi-tenant support --- Makefile | 2 +- README.md | 97 - charts/grafana/templates/_helpers.tmpl | 112 +- .../grafana/templates/grafana-configmap.yaml | 86 + .../grafana/templates/grafana-deployment.yaml | 36 + charts/grafana/templates/grafana-svc.yaml | 3 + charts/grafana/values.yaml | 4 +- rootfs/Dockerfile | 13 +- rootfs/usr/bin/start-grafana | 108 + .../dashboards/drycc_component_health.json | 2773 ----- .../api/dashboards/drycc_database.json | 1330 --- .../api/dashboards/drycc_registry.json | 1422 --- .../grafana/api/dashboards/drycc_storage.json | 3261 ------ .../grafana/api/dashboards/drycc_valkey.json | 1367 --- .../drycc_victoriametrics_cluster.json | 9923 ----------------- .../drycc_victoriametrics_vmagent.json | 7252 ------------ .../grafana/api/dashboards/kubernetes.json | 5929 ---------- .../api/dashboards/kubernetes_apiserver.json | 642 -- .../api/dashboards/kubernetes_cluster.json | 2013 ---- .../kubernetes_deployment_metrics.json | 1327 --- .../api/dashboards/kubernetes_metrics.json | 3393 ------ .../main/drycc_component_health.json | 2770 +++++ .../api/dashboards/main/drycc_database.json | 1327 +++ .../api/dashboards/main/drycc_registry.json | 1419 +++ .../api/dashboards/main/drycc_storage.json | 3258 ++++++ .../api/dashboards/main/drycc_valkey.json | 1364 +++ .../main/drycc_victoriametrics_cluster.json | 9920 ++++++++++++++++ .../main/drycc_victoriametrics_vmagent.json | 7249 ++++++++++++ .../api/dashboards/main/kubernetes.json | 5926 ++++++++++ .../dashboards/main/kubernetes_apiserver.json | 639 ++ .../dashboards/main/kubernetes_cluster.json | 2010 ++++ .../main/kubernetes_deployment_metrics.json | 1324 +++ .../dashboards/main/kubernetes_metrics.json | 3390 ++++++ .../grafana/api/dashboards/main/node.json | 3613 ++++++ .../share/grafana/api/dashboards/node.json | 3657 ------ .../api/dashboards/user/user_container.json | 2770 +++++ rootfs/usr/share/grafana/entrypoint.sh | 3 - rootfs/usr/share/grafana/grafana.ini.tpl | 261 - rootfs/usr/share/grafana/ldap.toml.tpl | 45 - rootfs/usr/share/grafana/start-grafana | 140 - 40 files changed, 47282 insertions(+), 44896 deletions(-) create mode 100644 charts/grafana/templates/grafana-configmap.yaml create mode 100755 rootfs/usr/bin/start-grafana delete mode 100644 rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json delete mode 100644 rootfs/usr/share/grafana/api/dashboards/drycc_database.json delete mode 100644 rootfs/usr/share/grafana/api/dashboards/drycc_registry.json delete mode 100644 rootfs/usr/share/grafana/api/dashboards/drycc_storage.json delete mode 100644 rootfs/usr/share/grafana/api/dashboards/drycc_valkey.json delete mode 100644 rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_cluster.json delete mode 100644 rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_vmagent.json delete mode 100644 rootfs/usr/share/grafana/api/dashboards/kubernetes.json delete mode 100644 rootfs/usr/share/grafana/api/dashboards/kubernetes_apiserver.json delete mode 100644 rootfs/usr/share/grafana/api/dashboards/kubernetes_cluster.json delete mode 100644 rootfs/usr/share/grafana/api/dashboards/kubernetes_deployment_metrics.json delete mode 100644 rootfs/usr/share/grafana/api/dashboards/kubernetes_metrics.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/main/drycc_component_health.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/main/drycc_database.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/main/drycc_registry.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/main/drycc_storage.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/main/drycc_valkey.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/main/drycc_victoriametrics_cluster.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/main/drycc_victoriametrics_vmagent.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/main/kubernetes.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/main/kubernetes_apiserver.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/main/kubernetes_cluster.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/main/kubernetes_deployment_metrics.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/main/kubernetes_metrics.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/main/node.json delete mode 100644 rootfs/usr/share/grafana/api/dashboards/node.json create mode 100644 rootfs/usr/share/grafana/api/dashboards/user/user_container.json delete mode 100644 rootfs/usr/share/grafana/entrypoint.sh delete mode 100644 rootfs/usr/share/grafana/grafana.ini.tpl delete mode 100644 rootfs/usr/share/grafana/ldap.toml.tpl delete mode 100755 rootfs/usr/share/grafana/start-grafana diff --git a/Makefile b/Makefile index b8b41c1..c10cfc8 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ SHELL := /bin/bash # grafana share/use the following targets/exports SHORT_NAME ?= grafana -SHELL_SCRIPTS = rootfs/usr/share/grafana/start-grafana +SHELL_SCRIPTS = rootfs/usr/bin/start-grafana BUILD_TAG ?= git-$(shell git rev-parse --short HEAD) DRYCC_REGISTRY ?= ${DEV_REGISTRY} diff --git a/README.md b/README.md index 215858a..0376504 100644 --- a/README.md +++ b/README.md @@ -13,103 +13,6 @@ We welcome your input! If you have feedback, please [submit an issue][issues]. I [Grafana](https://grafana.org/) is a graphing application built for time series data. It natively supports prometheus and provides great dashboarding support. This project is focused on provided a grafana installation that can be run within a kubernetes installation. The grafana application is agnostic to [Workflow][Workflow] and can be installed as a stand alone system with the monitoring suite. -## Configuration - -| ENV Var | Default Value | Description | -|---------|---------------|-------------| -| [APP_MODE](#app_mode) | development | Has two possible values (production or development). | -| [DATA_PATH](#data_path) | /var/lib/grafana | Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used). | -| [LOG_PATH](#log_path) | /var/log | Directory where grafana can store logs. | -| [SERVER_PROTOCOL](#server_protocol) | http | Protocol (http or https). | -| [BIND_ADDRESS](#bind_address) | `""` | The ip address to bind to, empty will bind to all interfaces | -| [BIND_PORT](#bind_port) | 3000 | The http port to use | -| [DOMAIN](#domain) | localhost | The public facing domain name used to access grafana from a browser | -| [ENFORCE_DOMAIN](#enforce_domain) | false | Redirect to correct domain if host header does not match domain. Prevents DNS rebinding attacks | -| [ROOT_URL](#root_url) | `"%(protocol)s://%(domain)s:%(http_port)s/"` | The full public facing url | -| [ROUTER_LOGGING](#router_logging) | false | Log web requests | -| [STATIC_ROOT_PATH](#static_root_path) | public | the path relative working path | -| [ENABLE_GZIP](#enable_gzip) | false | enable gzip | -| [CERT_FILE](#cert_file) | no default | https certs | -| [CERT_KEY](#cert_key) | no default | https certs key | -| [DATABASE_TYPE](#database_type) | no default | Potential values: mysql, postgres, sqlite3 | -| [DATABASE_HOST](#database_host) | no default | Address to external database host | -| [DATABASE_NAME](#database_name) | no default | Name of database to store information | -| [DATABASE_USER](#database_user) | no default | User to log into the database with | -| [DATABASE_PASSWORD](#database_password) | no default | Database password | -| [DATABASE_SSL_MODE](#database_ssl_mode) | no default | For "postgres" only, either "disable", "require" or "verify-full" | -| [DATABASE_PATH](#database_path) | no default | For "sqlite3" only, path relative to data_path setting | -| [SESSION_PROVIDER](#session_provider) | file | Either "memory", "file", "valkey", "mysql", "postgres", default is "file" | -| [SESSION_PROVIDER_CONFIG](#session_provider_config) | no default | Provider config options
`memory: not have any config yet`
`file: session dir path, is relative to grafana data_path`
`mysql: user:password@tcp(127.0.0.1:3306)/database_name`
`postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable` | -| [SESSION_COOKIE_NAME](#session_cookie_name) | grafana_sess | Session cookie name | -| [SESSION_LIFE_TIME](#session_life_time) | 86400 | Session life time, default is 86400 | -| [GRAFANA_REPORTING](#grafana_reporting) | false | Server reporting, sends usage counters to stats.grafana.org every 24 hours. No ip addresses are being tracked, only simple counters to track running instances, dashboard and error counts. It is very helpful to us. Change this option to false to disable reporting. | -| [GOOGLE_ANALYTICS_UA_ID](#google_analytics_ua_id) | no default | Google Analytics universal tracking code, only enabled if you specify an id here | -| [DEFAULT_USER](#default_user) | admin | default admin user, created on startup | -| [DEFAULT_USER_PASSWORD](#default_user_password) | admin | default admin password, can be changed before first start of grafana, or in profile settings | -| [SECRET_KEY](#secret_key) | SW2YcwTIb9zpOOhoPsMm | used for signing | -| [LOGIN_REMEMBER_DAYS](#login_remember_day) | 7 | Auto-login remember days | -| [COOKIE_USERNAME](#cookie_username) | grafana_user | | -| [COOKIE_REMEMBER_NAME](#cookie_remember_name) | grafana_remember | | -| [DISABLE_GRAVATAR](#disable_gravatar) | false | disable gravatar profile images | -| [DATASOURCE_PROXY_WHITELIST](#datasource_proxy_whitelist) | no default | data source proxy whitelist (ip_or_domain:port seperated by spaces) | -| [ALLOW_SIGN_UP](#allow_sign_up) | true | allow user signup / registration | -| [ALLOW_ORG_CREATE](#allow_org_create) | true | Allow non admin users to create organizations | -| [AUTO_ASSIGN_ORG](#auto_assign_org) | true | Set to true to automatically assign new users to the default organization (id 1) | -| [AUTO_ASSIGN_ORG_ROLE](#auto_assign_org_role) | Viewer | Default role new users will be automatically assigned (if disabled above is set to true) | -| [LOGIN_HINT](#login_hint) | email or username | Background text for the user field on the login page | -| [ANON_ACCESS](#anon_access) | false | enable anonymous access | -| [ORG_NAME](#org_name) | Main Org. | specify organization name that should be used for unauthenticated users | -| [UNAUTHED_USER_ORG_ROLE](#unauthed_user_org_role) | Viewer | specify role for unauthenticated users | -| [GITHUB_AUTH](#github_auth) | no default | Enable Github Auth (true/false) | -| [GITHUB_AUTH_ALLOW_SIGN_UP](#github_auth_allow_sign_up) | false | Allow signup with github auth | -| [GITHUB_AUTH_CLIENT_ID](#github_auth_client_id) | no default | Github Client Auth | -| [GITHUB_AUTH_CLIENT_SECRET](#github_auth_client_secret) | no default | Github Auth client secret | -| [GITHUB_AUTH_SCOPES](#github_auth_scopes) | user:email,read:org | Github Auth Scopes | -| [GITHUB_AUTH_URL](#github_auth_url) | https://github.com/login/oauth/authorize | Github authorization URL | -| [GITHUB_AUTH_TOKEN_URL](#github_auth_token_url) | https://github.com/login/oauth/access_token | Github Auth Token URL | -| [GITHUB_AUTH_API_URL](#github_auth_api_url) | https://api.github.com/user | Github Auth API URL | -| [GITHUB_AUTH_TEAM_IDS](#github_auth_team_ids) | "" | Team IDs to use for Github Auth | -| [GITHUB_AUTH_ALLOWED_ORGS](#github_auth_allowed_orgs) | "" | Allowed ORGs for Github Auth | -| [GOOGLE_AUTH](#google_auth) | no default | Enable Google Auth for login (true/false) | -| [GOOGLE_AUTH_ALLOW_SIGN_UP](#google_auth_allow_sign_up) | false | Allow people to sign up using Google Auth | -| [GOOGLE_AUTH_CLIENT_ID](#google_auth_client_id) | no default | Google Auth Client ID | -| [GOOGLE_AUTH_CLIENT_SECRET](#google_auth_client_secret) | no default | Google Auth Client Secret | -| [GOOGLE_AUTH_SCOPES](#google_auth_scopes) | https://www.googleapis.com/auth/userinfo.profile
https://www.googleapis.com/auth/userinfo.email | Google Auth Scopes | -| [GOOGLE_AUTH_URL](#google_auth_url) | https://accounts.google.com/o/oauth2/auth | Google Auth URL | -| [GOOGLE_AUTH_TOKEN_URL](#google_auth_token_url) | https://accounts.google.com/o/oauth2/token | Google Auth Token URL | -| [GOOGLE_AUTH_API_URL](#google_auth_api_url) | https://www.googleapis.com/oauth2/v1/userinfo | Google Auth API URL | -| [GOOGLE_AUTH_ALLOWED_DOMAINS](#google_auth_allowed_domains) | "" | Google Auth allowed domains | -| [AUTH_PROXY](#auth_proxy) | no default | If enabled use an authorization proxy | -| [AUTH_HEADER_NAME](#auth_header_name) | X-WEBAUTH-USER | Header to use for authorization | -| [AUTH_HEADER_PROPERTY](#auth_header_property) | username | Property to use in Authorization Header | -| [AUTH_AUTO_SIGN_UP](#auth_auto_sign_up) | true | Auto Signup user with data passed from Authorization Header| -| [BASIC_AUTH](#basic_auth) | true | Enable Basic Auth | -| [LDAP_AUTH](#ldap_auth) | no default | Enable LDAP Auth | -| [LDAP_AUTH_CONFIG_FILE](#ldap_auth_config_file) | /etc/grafana/ldap.toml | Config file to use for LDAP Auth | -| [SMTP](#smtp) | no default | Enable SMTP/Emailing | -| [SMTP_HOST](#smtp_host) | no default | SMTP Host | -| [SMTP_USER](#smtp_user) | no default | User for SMTP | -| [SMTP_PASSWORD](#smtp_password) | no default | Password for SMTP | -| [SMTP_CERT_FILE](#smtp_cert_file) | no default | Cert file for SMTP HTTPS | -| [SMTP_KEY_FILE](#smtp_key_file) | no default | Key file for SMTP HTTPS | -| [SMTP_SKIP_VERIFY](#smtp_skip_verify) | false | Skip HTTPS verify | -| [SMTP_FROM_ADDRESS](#smtp_from_address) | admin@grafana.localhost | From address to use when sending emails | -| [WELCOME_EMAIL](#welcome_email) | false | Send welcome email on sign up | -| [LOG_MODE](#log_mode) | console | Either "console", "file", default is "console" Use comma to separate multiple modes, e.g. "console, file" | -| [BUFFER_LENGTH](#buffer_length) | 10000 | Buffer length of channel, keep it as it is if you don't know what it is. | -| [LOG_LEVEL](#log_level) | Info | Either "Trace", "Debug", "Info", "Warn", "Error", "Critical", default is "Trace" | -| [LOG_LEVEL_CONSOLE](#log_level_console) | Info | For "console" mode only | -| [LOG_LEVEL_FILE](#log_level_file) | Info | For "file" mode only | -| [LOG_ROTATE](#log_rotate) | true | This enables automated log rotate(switch of following options), default is true | -| [LOG_MAX_LINES](#log_max_lines) | 1000000 | Max line number of single file | -| [LOG_MAX_LINES_SHIFT](#log_max_lines_shift) | 28 | Max size shift of single file, default is 28 means 1 << 28, 256MB | -| [LOG_DAILY_ROTATE](#log_daily_rotate) | true | Segment log daily, default is true | -| [LOG_MAX_DAYS](#log_max_days) | 7 | Expired days of log file(delete after max days) | -| [DASHBOARD_JSON](#dashboard_json) | true | Poll a location for json files that contain dashboards | -| [DASHBOARD_JSON_PATH](#dashboard_json_path) | /usr/share/grafana/dashboards | Location to scan for json dashboards | -| [PLUGINS_PATH](#plugins_path) | /var/lib/grafana/plugins | Path to where grafana can install plugins | -| [GF_INSTALL_PLUGINS](#gf_install_plugins) | no default | Pass the plugins as a comma seperated list | - ## Development The provided `Makefile` has various targets to help support building and publishing new images into a kubernetes cluster. diff --git a/charts/grafana/templates/_helpers.tmpl b/charts/grafana/templates/_helpers.tmpl index 1e195ff..9869f8d 100644 --- a/charts/grafana/templates/_helpers.tmpl +++ b/charts/grafana/templates/_helpers.tmpl @@ -1,10 +1,39 @@ {{/* Generate grafana envs */}} {{- define "grafana.envs" }} env: +- name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace +{{- if .Values.prometheusUrl }} +- name: "PROMETHEUS_URL" + value: "{{ .Values.prometheusUrl }}" +{{- else if .Values.prometheus.enabled }} +- name: "PROMETHEUS_URL" + value: "http://$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)/v2/prometheus" +{{- end }} +- name: "VALKEY_LOCATION" + value: {{ ternary "on-cluster" "off-cluster" .Values.valkey.enabled }} +- name: "STORAGE_LOCATION" + value: {{ ternary "on-cluster" "off-cluster" .Values.storage.enabled }} +- name: "DATABASE_LOCATION" + value: {{ ternary "on-cluster" "off-cluster" .Values.database.enabled }} +- name: "PASSPORT_LOCATION" + value: {{ ternary "on-cluster" "off-cluster" .Values.passport.enabled }} +- name: "REGISTRY_LOCATION" + value: {{ ternary "on-cluster" "off-cluster" .Values.registry.enabled }} +- name: "PROMETHEUS_LOCATION" + value: {{ ternary "on-cluster" "off-cluster" .Values.prometheus.enabled }} +- name: "DRYCC_GATEWAY_SCHEME" +{{- if .Values.global.certManagerEnabled }} + value: https +{{- else }} + value: http +{{- end}} - name: GF_DATABASE_TYPE value: postgres {{- if (.Values.databaseUrl) }} @@ -24,70 +53,55 @@ env: - name: GF_DATABASE_URL value: "postgres://$(GF_DATABASE_USER):$(GF_DATABASE_PASSWORD)@drycc-database.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:5432/grafana" {{- end }} -{{- if .Values.prometheusUrl }} -- name: "PROMETHEUS_URL" - value: "{{ .Values.prometheusUrl }}" -{{- else if .Values.prometheus.enabled }} -- name: "PROMETHEUS_URL" - value: "http://drycc-victoriametrics-vmselect.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:8481/select/0/prometheus" -{{- end }} -- name: "BIND_PORT" - value: "3000" -{{- if not (.Values.environment.DEFAULT_USER) }} -- name: "DEFAULT_USER" +{{- if not (.Values.environment.GF_SECURITY_ADMIN_USER) }} +- name: "GF_SECURITY_ADMIN_USER" value: {{ randAlphaNum 32 }} {{- end}} -{{- if not (.Values.environment.DEFAULT_USER_PASSWORD) }} -- name: "DEFAULT_USER_PASSWORD" +{{- if not (.Values.environment.GF_SECURITY_ADMIN_PASSWORD) }} +- name: "GF_SECURITY_ADMIN_PASSWORD" value: {{ randAlphaNum 32 }} {{- end}} -- name: "KUBERNETES_CLUSTER_DOMAIN" - value: {{.Values.global.clusterDomain}} -{{- range $key, $value := .Values.environment }} -- name: {{ $key }} - value: {{ $value | quote }} -{{- end }} -- name: "DRYCC_GRAFANA_URL" -{{- if .Values.global.certManagerEnabled }} - value: https://drycc-grafana.{{ .Values.global.platformDomain }} -{{- else }} - value: http://drycc-grafana.{{ .Values.global.platformDomain }} -{{- end }} +{{- if not (.Values.environment.GF_SECURITY_SECRET_KEY) }} +- name: "GF_SECURITY_SECRET_KEY" + value: {{ randAlphaNum 32 }} +{{- end}} +- name: "GF_SERVER_ROOT_URL" + value: $(DRYCC_GATEWAY_SCHEME)://drycc-grafana.{{ .Values.global.platformDomain }} {{- if .Values.passport.enabled}} -- name: "DRYCC_PASSPORT_URL" -{{- if .Values.global.certManagerEnabled }} - value: https://drycc-passport.{{ .Values.global.platformDomain }} -{{- else }} - value: http://drycc-passport.{{ .Values.global.platformDomain }} -{{- end }} -- name: DRYCC_PASSPORT_KEY +- name: GF_AUTH_GENERIC_OAUTH_AUTH_URL + value: $(DRYCC_GATEWAY_SCHEME)://drycc-passport.{{ .Values.global.platformDomain }}/oauth/authorize/ +- name: GF_AUTH_GENERIC_OAUTH_TOKEN_URL + value: $(DRYCC_GATEWAY_SCHEME)://drycc-passport.{{ .Values.global.platformDomain }}/oauth/token/ +- name: GF_AUTH_GENERIC_OAUTH_API_URL + value: $(DRYCC_GATEWAY_SCHEME)://drycc-passport.{{ .Values.global.platformDomain }}/oauth/userinfo/ +- name: GF_AUTH_GENERIC_OAUTH_CLIENT_ID valueFrom: secretKeyRef: name: passport-creds key: drycc-passport-grafana-key -- name: DRYCC_PASSPORT_SECRET +- name: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET valueFrom: secretKeyRef: name: passport-creds key: drycc-passport-grafana-secret {{- else }} -- name: DRYCC_PASSPORT_URL - value: "{{ .Values.passportUrl }}" -- name: DRYCC_PASSPORT_KEY +- name: GF_AUTH_GENERIC_OAUTH_AUTH_URL + value: {{ .Values.passportUrl }}/oauth/authorize/ +- name: GF_AUTH_GENERIC_OAUTH_TOKEN_URL + value: {{ .Values.passportUrl }}/oauth/token/ +- name: GF_AUTH_GENERIC_OAUTH_API_URL + value: {{ .Values.passportUrl }}/oauth/userinfo/ +- name: GF_AUTH_GENERIC_OAUTH_CLIENT_ID value: "{{ .Values.passportKey }}" -- name: DRYCC_PASSPORT_SECRET +- name: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET value: "{{ .Values.passportSecret }}" {{- end }} -- name: "VALKEY_LOCATION" - value: {{ ternary "on-cluster" "off-cluster" .Values.valkey.enabled }} -- name: "STORAGE_LOCATION" - value: {{ ternary "on-cluster" "off-cluster" .Values.storage.enabled }} -- name: "DATABASE_LOCATION" - value: {{ ternary "on-cluster" "off-cluster" .Values.database.enabled }} -- name: "PASSPORT_LOCATION" - value: {{ ternary "on-cluster" "off-cluster" .Values.passport.enabled }} -- name: "REGISTRY_LOCATION" - value: {{ ternary "on-cluster" "off-cluster" .Values.registry.enabled }} -- name: "PROMETHEUS_LOCATION" - value: {{ ternary "on-cluster" "off-cluster" .Values.prometheus.enabled }} +- name: GF_UNIFIED_ALERTING_HA_PEERS + value: "drycc-grafana.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:9094" +- name: GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS + value: "$(POD_IP):9094" +{{- range $key, $value := .Values.environment }} +- name: {{ $key }} + value: {{ $value | quote }} +{{- end }} {{- end }} diff --git a/charts/grafana/templates/grafana-configmap.yaml b/charts/grafana/templates/grafana-configmap.yaml new file mode 100644 index 0000000..7b761a7 --- /dev/null +++ b/charts/grafana/templates/grafana-configmap.yaml @@ -0,0 +1,86 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-config + namespace: {{ .Release.Namespace }} + labels: + heritage: drycc +data: + grafana.ini: | + app_mode = production + + [paths] + data = /var/lib/grafana + home = /usr/share/grafana + logs = /var/log/grafana + plugins = /var/lib/grafana/plugins + provisioning = /opt/drycc/grafana/conf/provisioning + + [server] + http_port = 3000 + router_logging = false + static_root_path = public + enable_gzip = true + + [analytics] + reporting_enabled = false + + [security] + admin_user = drycc + admin_password = drycc + admin_email = admin@drycc.cc + login_remember_days = 7 + cookie_username = grafana_user + cookie_remember_name = grafana_remember + disable_gravatar = false + + [users] + allow_sign_up = false + allow_org_create = false + auto_assign_org = true + auto_assign_org_id = 1 + login_hint = email or username + + [auth] + disable_login_form = true + + [auth.anonymous] + enabled = false + org_name = Main Org. + + [auth.basic] + enabled = true + + [auth.generic_oauth] + enabled = true + name = Drycc OAuth + auto_login = true + scopes = email,profile,openid + allow_sign_up = true + tls_skip_verify_insecure = true + org_attribute_path = roles + org_mapping = admin:*:Admin staff:*:Editor users:2:Viewer + role_attribute_path = (is_active && ((is_superuser && 'GrafanaAdmin') || (is_staff && 'Editor'))) || 'None' + allow_assign_grafana_admin = true + + [emails] + welcome_email_on_sign_up = false + + [log] + mode = console + buffer_len = 10000 + level = info + + [log.console] + level = info + + [unified_alerting] + enabled = true + disabled_orgs = 2 + ha_peer_timeout = 15s + ha_reconnect_timeout = 2m + ha_listen_address = 0.0.0.0:9094 + + [dashboards.json] + enabled = true + path = /usr/share/grafana/dashboards diff --git a/charts/grafana/templates/grafana-deployment.yaml b/charts/grafana/templates/grafana-deployment.yaml index b4fb98b..b3fc564 100644 --- a/charts/grafana/templates/grafana-deployment.yaml +++ b/charts/grafana/templates/grafana-deployment.yaml @@ -54,13 +54,49 @@ spec: ports: - containerPort: 3000 name: grafana + - containerPort: 9094 + name: alerting {{- include "grafana.envs" . | indent 8 }} + livenessProbe: + tcpSocket: + port: grafana + initialDelaySeconds: 120 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + readinessProbe: + httpGet: + path: /api/health + port: grafana + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + startupProbe: + httpGet: + path: /api/health + port: grafana + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 {{- if .Values.persistence.enabled }} volumeMounts: - name: grafana-data mountPath: /var/lib/grafana + - name: grafana-config + readOnly: false + subPath: grafana.ini + readOnly: true + mountPath: /opt/drycc/grafana/conf/grafana.ini volumes: - name: grafana-data persistentVolumeClaim: claimName: drycc-grafana + - name: grafana-config + configMap: + name: grafana-config {{- end }} diff --git a/charts/grafana/templates/grafana-svc.yaml b/charts/grafana/templates/grafana-svc.yaml index 30e0854..15dbdb1 100644 --- a/charts/grafana/templates/grafana-svc.yaml +++ b/charts/grafana/templates/grafana-svc.yaml @@ -14,5 +14,8 @@ spec: - port: 80 name: grafana targetPort: grafana + - port: 9094 + name: alerting + targetPort: alerting selector: app: drycc-grafana diff --git a/charts/grafana/values.yaml b/charts/grafana/values.yaml index 09ed45e..cec250b 100644 --- a/charts/grafana/values.yaml +++ b/charts/grafana/values.yaml @@ -54,8 +54,8 @@ persistence: # Any custom grafana environment variables # can be specified as key-value pairs under environment # this is usually a non required setting. -environment: - ALLOW_SIGN_UP: false +environment: {} + ## databaseUrl and databaseReplicaUrl are will no longer use the built-in database component databaseUrl: "" # The passportUrl, passportKey and passportSecret are will no longer use the built-in passport component diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 3b69504..48fcbac 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -4,8 +4,7 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/usr/share/grafana \ - GRAFANA_VERSION="12.0.0" \ - ENVTPL_VERSION="1.0.0" \ + GRAFANA_VERSION="12.0.1" \ JQ_VERSION="1.7.1" RUN groupadd drycc --gid ${DRYCC_GID} \ @@ -13,9 +12,8 @@ RUN groupadd drycc --gid ${DRYCC_GID} \ COPY . / -RUN install-packages fontconfig curl \ +RUN install-packages fontconfig \ && install-stack grafana ${GRAFANA_VERSION} \ - && install-stack envtpl ${ENVTPL_VERSION} \ && install-stack jq ${JQ_VERSION} \ && rm -rf \ /usr/share/doc \ @@ -32,12 +30,9 @@ RUN install-packages fontconfig curl \ /usr/lib/`echo $(uname -m)`-linux-gnu/gconv/EBC* \ && mkdir -p /usr/share/man/man{1..8} \ && mkdir /var/lib/grafana -p \ - && chown -R ${DRYCC_UID}:${DRYCC_GID} /opt/drycc/grafana /usr/share/grafana /var/lib/grafana \ - && chmod +x /usr/share/grafana/start-grafana \ - && chmod 644 /usr/share/grafana/grafana.ini.tpl \ - && chmod +x /usr/share/grafana/entrypoint.sh + && chown -R ${DRYCC_UID}:${DRYCC_GID} /opt/drycc/grafana /usr/share/grafana /var/lib/grafana USER ${DRYCC_UID} WORKDIR ${DRYCC_HOME_DIR} -CMD ["./start-grafana"] +CMD ["start-grafana"] EXPOSE 3000 diff --git a/rootfs/usr/bin/start-grafana b/rootfs/usr/bin/start-grafana new file mode 100755 index 0000000..8d96a0d --- /dev/null +++ b/rootfs/usr/bin/start-grafana @@ -0,0 +1,108 @@ +#!/bin/bash +shopt -s expand_aliases + +alias curl="curl -s -H 'Content-Type: application/json' -H 'Accept: application/json'" +NAMESPACE=${NAMESPACE:-drycc} +GRAFANA_API_URL="http://${GF_SECURITY_ADMIN_USER}:${GF_SECURITY_ADMIN_PASSWORD}@localhost:${GF_SERVER_HTTP_PORT:-3000}" + +set -m +echo "Starting Grafana in the background" +exec grafana server --config /opt/drycc/grafana/conf/grafana.ini --homepath /opt/drycc/grafana & +echo "Waiting for Grafana to come up..." +until curl -q --fail --output /dev/null --silent "${GRAFANA_API_URL}/api/org"; do + printf "." + sleep 2 +done +echo "Grafana is up and running." + +# Create organization. if it doesn't exist +echo "Creating user organization..." +curl -XPOST "${GRAFANA_API_URL}/api/orgs" -d '{"name":"User Org."}' + +# Get the JSON for the all organization +GRAFANA_MAIN_ORG_ID=$(curl -XGET "${GRAFANA_API_URL}/api/orgs" | jq '.[] | select(.name == "Main Org.") | .id') +GRAFANA_USER_ORG_ID=$(curl -XGET "${GRAFANA_API_URL}/api/orgs" | jq '.[] | select(.name == "User Org.") | .id') + +# Prometheus Service Host and Port +echo "Using the following URL for prometheus: ${PROMETHEUS_URL}" +echo "Creating main organization prometheus datasource..." +curl -XPOST -H "X-Grafana-Org-Id: ${GRAFANA_MAIN_ORG_ID}" "${GRAFANA_API_URL}/api/datasources" -d ' +{ + "name": "prometheus-datasource", + "type": "prometheus", + "access": "proxy", + "isDefault": true, + "url": "'"${PROMETHEUS_URL}"'", + "jsonData": {"httpMethod": "GET", "oauthPassThru": true, "sigV4Auth": false} +}' +echo "Creating user organization prometheus datasource..." +curl -XPOST -H "X-Grafana-Org-Id: ${GRAFANA_USER_ORG_ID}" "${GRAFANA_API_URL}/api/datasources" -d ' +{ + "name": "prometheus-datasource", + "type": "prometheus", + "access": "proxy", + "isDefault": true, + "url": "'"${PROMETHEUS_URL}"'", + "jsonData": {"httpMethod": "GET", "oauthPassThru": true, "sigV4Auth": false} +}' + +echo "" +echo "Importing default dashboards..." +DASHBOARD_LOCATION="/usr/share/grafana/api/dashboards" +# replace namespace +# shellcheck disable=SC1003 +# shellcheck disable=SC2046 +sed -i 's@r\[\\"namespace\\\"\] == \\"drycc\\"@r\[\\"namespace\\"\] == \\"\'"${NAMESPACE}"'\\"@g' $(grep -F 'r[\"namespace\"] == \"drycc\"' -rl --include="*.json" "${DASHBOARD_LOCATION}/main") +# drycc component dashboard +DCD="${DASHBOARD_LOCATION}"/main/drycc_component_health.json + +# remove off-cluster component panel +if [ "on-cluster" != "${VALKEY_LOCATION}" ]; then + rm -rf "${DASHBOARD_LOCATION}"/main/drycc_valkey.json + # shellcheck disable=SC2005 + echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "VALKEY"))')" > "${DCD}" +fi +if [ "on-cluster" != "${DATABASE_LOCATION}" ]; then + rm -rf "${DASHBOARD_LOCATION}"/main/drycc_database.json + # shellcheck disable=SC2005 + echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "DATABASE"))')" > "${DCD}" +fi +if [ "on-cluster" != "${PASSPORT_LOCATION}" ]; then + # shellcheck disable=SC2005 + echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "PASSPORT"))')" > "${DCD}" +fi +if [ "on-cluster" != "${REGISTRY_LOCATION}" ]; then + rm -rf "${DASHBOARD_LOCATION}"/main/drycc_registry.json + # shellcheck disable=SC2005 + echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "REGISTRY"))')" > "${DCD}" + # shellcheck disable=SC2005 + echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "REGISTRY-PROXY"))')" > "${DCD}" +fi +if [ "on-cluster" != "${STORAGE_LOCATION}" ]; then + rm -rf "${DASHBOARD_LOCATION}"/main/drycc_storage.json + # shellcheck disable=SC2005 + echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "STORAGE"))')" > "${DCD}" +fi + +if [ "on-cluster" != "${PROMETHEUS_LOCATION}" ]; then + rm -rf "${DASHBOARD_LOCATION}"/main/drycc_victoriametrics_*.json +fi + +echo "Creating main organization grafana dashboard..." +for filename in "${DASHBOARD_LOCATION}"/main/*.json; do + echo "Importing ${filename} ..." + jq '{overwrite: true, dashboard: .}' "${filename}" | curl -XPOST --data-binary @- -H "X-Grafana-Org-Id: ${GRAFANA_MAIN_ORG_ID}" "${GRAFANA_API_URL}/api/dashboards/db" + echo "" + echo "Done importing ${filename}" +done + +echo "Creating main organization grafana dashboard..." +for filename in "${DASHBOARD_LOCATION}"/user/*.json; do + echo "Importing ${filename} ..." + jq '{overwrite: true, dashboard: .}' "${filename}" | curl -XPOST --data-binary @- -H "X-Grafana-Org-Id: ${GRAFANA_USER_ORG_ID}" "${GRAFANA_API_URL}/api/dashboards/db" + echo "" + echo "Done importing ${filename}" +done + +echo "Bringing Grafana back to the foreground" +fg diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json b/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json deleted file mode 100644 index b8046bd..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_component_health.json +++ /dev/null @@ -1,2773 +0,0 @@ -{ - "dashboard": { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 33, - "panels": [], - "targets": [ - { - "refId": "A" - } - ], - "title": "SUM", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 1 - }, - "hiddenSeries": false, - "id": 31, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_namespace", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",container!=\"\",namespace=\"drycc\"}[1m])) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["namespace"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total CPU Usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 1 - }, - "hiddenSeries": false, - "id": 32, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_namespace", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",container!=\"\",namespace=\"drycc\"})", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["namespace"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total Memory Usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 52, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 2 - }, - "hiddenSeries": false, - "id": 54, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",container=~\"^drycc-passport.*$\"}[1m])) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "hide": false, - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 2 - }, - "hiddenSeries": false, - "id": 56, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-passport.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "PASSPORT", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 34, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 3 - }, - "hiddenSeries": false, - "id": 1, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-controller.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "hide": false, - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 3 - }, - "hiddenSeries": false, - "id": 2, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-controller.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "CONTROLLER", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 35, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 4 - }, - "hiddenSeries": false, - "id": 3, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-builder.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 4 - }, - "hiddenSeries": false, - "id": 4, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-builder.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "BUILDER", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 36, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 12 - }, - "hiddenSeries": false, - "id": 6, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-database.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 12 - }, - "hiddenSeries": false, - "id": 5, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-database.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "DATABASE", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 37, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 13 - }, - "hiddenSeries": false, - "id": 19, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-logger-fluentbit.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 13 - }, - "hiddenSeries": false, - "id": 20, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-logger-fluentbit.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "FLUENTBIT", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 38, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 7 - }, - "hiddenSeries": false, - "id": 25, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "datasource": null, - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-grafana.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 7 - }, - "hiddenSeries": false, - "id": 26, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "datasource": null, - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-grafana.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "GRAFANA", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 40, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 16 - }, - "hiddenSeries": false, - "id": 7, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=\"drycc-logger\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 16 - }, - "hiddenSeries": false, - "id": 8, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=\"drycc-logger\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "LOGGER", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 41, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 17 - }, - "hiddenSeries": false, - "id": 9, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 17 - }, - "hiddenSeries": false, - "id": 10, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "STORAGE", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 43, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 21 - }, - "hiddenSeries": false, - "id": 27, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-valkey.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 21 - }, - "hiddenSeries": false, - "id": 28, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-valkey.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "VALKEY", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 44, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 22 - }, - "hiddenSeries": false, - "id": 11, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-registry.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 22 - }, - "hiddenSeries": false, - "id": 12, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=\"drycc-registry\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "REGISTRY", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 45, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 23 - }, - "hiddenSeries": false, - "id": 16, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-registry-proxy.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 23 - }, - "hiddenSeries": false, - "id": 15, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-registry-proxy.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "REGISTRY-PROXY", - "type": "row" - } - ], - "refresh": "5s", - "schemaVersion": 37, - "style": "dark", - "tags": ["drycc"], - "templating": { - "list": [] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] - }, - "timezone": "browser", - "title": "Drycc Component Health", - "uid": "2ySBrTTMz", - "version": 1, - "weekStart": "" - }, - "overwrite": true -} diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_database.json b/rootfs/usr/share/grafana/api/dashboards/drycc_database.json deleted file mode 100644 index bf40df8..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_database.json +++ /dev/null @@ -1,1330 +0,0 @@ -{ - "dashboard": { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "A quickstart to setup the Prometheus PostgreSQL Exporter with preconfigured dashboards, alerting rules, and recording rules.", - "editable": true, - "gnetId": 14114, - "graphTooltip": 0, - "id": 1, - "iteration": 1603191461722, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 20, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 1, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "fetched", - "dsType": "prometheus", - "expr": "sum(irate(pg_stat_database_tup_fetched{datname=~\"$db\",instance=~\"$instance\"}[5m]))", - "format": "time_series", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "intervalFactor": 2, - "legendFormat": "fetched", - "measurement": "postgresql", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": ["tup_fetched"], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": ["10s"], - "type": "non_negative_derivative" - } - ] - ], - "step": 120, - "tags": [ - { - "key": "instance", - "operator": "=~", - "value": "/^$instance$/" - } - ] - }, - { - "alias": "fetched", - "dsType": "prometheus", - "expr": "sum(irate(pg_stat_database_tup_returned{datname=~\"$db\",instance=~\"$instance\"}[5m]))", - "format": "time_series", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "intervalFactor": 2, - "legendFormat": "returned", - "measurement": "postgresql", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": ["tup_fetched"], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": ["10s"], - "type": "non_negative_derivative" - } - ] - ], - "step": 120, - "tags": [ - { - "key": "instance", - "operator": "=~", - "value": "/^$instance$/" - } - ] - }, - { - "alias": "fetched", - "dsType": "prometheus", - "expr": "sum(irate(pg_stat_database_tup_inserted{datname=~\"$db\",instance=~\"$instance\"}[5m]))", - "format": "time_series", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "intervalFactor": 2, - "legendFormat": "inserted", - "measurement": "postgresql", - "policy": "default", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": ["tup_fetched"], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": ["10s"], - "type": "non_negative_derivative" - } - ] - ], - "step": 120, - "tags": [ - { - "key": "instance", - "operator": "=~", - "value": "/^$instance$/" - } - ] - }, - { - "alias": "fetched", - "dsType": "prometheus", - "expr": "sum(irate(pg_stat_database_tup_updated{datname=~\"$db\",instance=~\"$instance\"}[5m]))", - "format": "time_series", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "intervalFactor": 2, - "legendFormat": "updated", - "measurement": "postgresql", - "policy": "default", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": ["tup_fetched"], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": ["10s"], - "type": "non_negative_derivative" - } - ] - ], - "step": 120, - "tags": [ - { - "key": "instance", - "operator": "=~", - "value": "/^$instance$/" - } - ] - }, - { - "alias": "fetched", - "dsType": "prometheus", - "expr": "sum(irate(pg_stat_database_tup_deleted{datname=~\"$db\",instance=~\"$instance\"}[5m]))", - "format": "time_series", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "intervalFactor": 2, - "legendFormat": "deleted", - "measurement": "postgresql", - "policy": "default", - "refId": "E", - "resultFormat": "time_series", - "select": [ - [ - { - "params": ["tup_fetched"], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": ["10s"], - "type": "non_negative_derivative" - } - ] - ], - "step": 120, - "tags": [ - { - "key": "instance", - "operator": "=~", - "value": "/^$instance$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Rows", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 0, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 0 - }, - "height": "55px", - "id": 11, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "dsType": "prometheus", - "expr": "sum(irate(pg_stat_database_xact_commit{datname=~\"$db\",instance=~\"$instance\"}[5m])) + sum(irate(pg_stat_database_xact_rollback{datname=~\"$db\",instance=~\"$instance\"}[5m]))", - "format": "time_series", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "intervalFactor": 2, - "measurement": "postgresql", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": ["xact_commit"], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": ["10s"], - "type": "non_negative_derivative" - } - ] - ], - "step": 1800, - "tags": [ - { - "key": "instance", - "operator": "=~", - "value": "/^$instance$/" - } - ] - } - ], - "thresholds": "", - "title": "QPS", - "transparent": true, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "decimals": 1, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 7 - }, - "hiddenSeries": false, - "id": 2, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "hideZero": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Buffers Allocated", - "dsType": "prometheus", - "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance=~'$instance'}[5m])", - "format": "time_series", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "intervalFactor": 2, - "legendFormat": "buffers_alloc", - "measurement": "postgresql", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": ["buffers_alloc"], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [], - "type": "difference" - } - ] - ], - "step": 240, - "tags": [ - { - "key": "instance", - "operator": "=~", - "value": "/^$instance$/" - } - ] - }, - { - "alias": "Buffers Allocated", - "dsType": "prometheus", - "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{instance=~'$instance'}[5m])", - "format": "time_series", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "intervalFactor": 2, - "legendFormat": "buffers_backend_fsync", - "measurement": "postgresql", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": ["buffers_alloc"], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [], - "type": "difference" - } - ] - ], - "step": 240, - "tags": [ - { - "key": "instance", - "operator": "=~", - "value": "/^$instance$/" - } - ] - }, - { - "alias": "Buffers Allocated", - "dsType": "prometheus", - "expr": "irate(pg_stat_bgwriter_buffers_backend{instance=~'$instance'}[5m])", - "format": "time_series", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "intervalFactor": 2, - "legendFormat": "buffers_backend", - "measurement": "postgresql", - "policy": "default", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": ["buffers_alloc"], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [], - "type": "difference" - } - ] - ], - "step": 240, - "tags": [ - { - "key": "instance", - "operator": "=~", - "value": "/^$instance$/" - } - ] - }, - { - "alias": "Buffers Allocated", - "dsType": "prometheus", - "expr": "irate(pg_stat_bgwriter_buffers_clean{instance=~'$instance'}[5m])", - "format": "time_series", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "intervalFactor": 2, - "legendFormat": "buffers_clean", - "measurement": "postgresql", - "policy": "default", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": ["buffers_alloc"], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [], - "type": "difference" - } - ] - ], - "step": 240, - "tags": [ - { - "key": "instance", - "operator": "=~", - "value": "/^$instance$/" - } - ] - }, - { - "alias": "Buffers Allocated", - "dsType": "prometheus", - "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{instance=~'$instance'}[5m])", - "format": "time_series", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "intervalFactor": 2, - "legendFormat": "buffers_checkpoint", - "measurement": "postgresql", - "policy": "default", - "refId": "E", - "resultFormat": "time_series", - "select": [ - [ - { - "params": ["buffers_alloc"], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [], - "type": "difference" - } - ] - ], - "step": 240, - "tags": [ - { - "key": "instance", - "operator": "=~", - "value": "/^$instance$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Buffers", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 7 - }, - "hiddenSeries": false, - "id": 3, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "conflicts", - "dsType": "prometheus", - "expr": "sum(rate(pg_stat_database_deadlocks{datname=~\"$db\",instance=~\"$instance\"}[5m]))", - "format": "time_series", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "intervalFactor": 2, - "legendFormat": "deadlocks", - "measurement": "postgresql", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": ["conflicts"], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [], - "type": "difference" - } - ] - ], - "step": 240, - "tags": [ - { - "key": "instance", - "operator": "=~", - "value": "/^$instance$/" - } - ] - }, - { - "alias": "deadlocks", - "dsType": "prometheus", - "expr": "sum(rate(pg_stat_database_conflicts{datname=~\"$db\",instance=~\"$instance\"}[5m]))", - "format": "time_series", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "intervalFactor": 2, - "legendFormat": "conflicts", - "measurement": "postgresql", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": ["deadlocks"], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [], - "type": "difference" - } - ] - ], - "step": 240, - "tags": [ - { - "key": "instance", - "operator": "=~", - "value": "/^$instance$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Conflicts/Deadlocks", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 14 - }, - "hiddenSeries": false, - "id": 12, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": true, - "pluginVersion": "7.2.1", - "pointradius": 1, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=~\"$instance\"}) / (sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=~\"$instance\"}) + sum(pg_stat_database_blks_read{datname=~\"$db\",instance=~\"$instance\"}))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "cache hit rate", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Cache hit ratio", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 14 - }, - "hiddenSeries": false, - "id": 13, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pg_stat_database_numbackends{datname=~\"$db\",instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{__name__}}", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Number of active connections", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": false, - "schemaVersion": 26, - "style": "dark", - "tags": ["postgres"], - "templating": { - "list": [ - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": null, - "multi": false, - "name": "instance", - "options": [], - "query": "label_values(up{job=~\"postgres.*\"},instance)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "definition": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", - "hide": 0, - "includeAll": true, - "label": "db", - "multi": false, - "name": "db", - "options": [], - "query": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "Postgres Overview", - "value": "Postgres Overview" - }, - "hide": 0, - "includeAll": false, - "label": "datasource", - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": null, - "current": { - "selected": true, - "text": "postgres", - "value": "postgres" - }, - "definition": "label_values(pg_up, job)", - "hide": 0, - "includeAll": false, - "label": "job", - "multi": false, - "name": "job", - "options": [ - { - "selected": true, - "text": "postgres", - "value": "postgres" - } - ], - "query": "label_values(pg_up, job)", - "refresh": 0, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] - }, - "timezone": "browser", - "title": "PostgreSQL Exporter Quickstart and Dashboard", - "uid": "wGgaPlciz", - "version": 5 - }, - "overwrite": true -} diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_registry.json b/rootfs/usr/share/grafana/api/dashboards/drycc_registry.json deleted file mode 100644 index 5a5cf38..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_registry.json +++ /dev/null @@ -1,1422 +0,0 @@ -{ - "dashboard": { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": 9621, - "graphTooltip": 0, - "id": null, - "iteration": 1547215355938, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 22, - "panels": [], - "title": "General Counters, CPU, Memory and File Descriptor Stats", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], - "description": "start time of the process", - "format": "dateTimeFromNow", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 4, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "110%", - "prefix": "", - "prefixFontSize": "110%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "process_start_time_seconds{app=\"$app\", instance=\"$instance\"} * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Start Time", - "type": "singlestat", - "valueFontSize": "110%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "cacheTimeout": null, - "colorBackground": true, - "colorValue": false, - "colors": ["#967302", "#967302", "#967302"], - "description": "The number of cache request received", - "format": "short", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 4, - "y": 1 - }, - "id": 16, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "150%", - "prefix": "", - "prefixFontSize": "150%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "#eab839", - "full": false, - "lineColor": "#e5ac0e", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "registry_storage_cache_total{instance=\"$instance\",type=\"Request\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ type }}", - "refId": "A" - } - ], - "thresholds": "", - "title": "Total Cache Requests", - "type": "singlestat", - "valueFontSize": "150%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "max" - }, - { - "cacheTimeout": null, - "colorBackground": true, - "colorValue": false, - "colors": ["#1f78c1", "#1f78c1", "#1f78c1"], - "description": "The number of cache request received (hits)", - "format": "short", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 9, - "y": 1 - }, - "id": 12, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "150%", - "prefix": "", - "prefixFontSize": "150%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "#5195ce", - "full": false, - "lineColor": "#0a50a1", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "registry_storage_cache_total{instance=\"$instance\",type=\"Hit\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ type }}", - "refId": "A" - } - ], - "thresholds": "", - "title": "Total Hit Cache Requests", - "type": "singlestat", - "valueFontSize": "150%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "max" - }, - { - "cacheTimeout": null, - "colorBackground": true, - "colorValue": false, - "colors": ["#890f02", "#890f02", "#890f02"], - "description": "The number of cache request received (miss)", - "format": "short", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 14, - "y": 1 - }, - "id": 14, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "150%", - "prefix": "", - "prefixFontSize": "150%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "#e24d42", - "full": false, - "lineColor": "#e24d42", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "registry_storage_cache_total{instance=\"$instance\",type=\"Miss\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ type }}", - "refId": "A" - } - ], - "thresholds": "", - "title": "Total Missed Cache Requests", - "type": "singlestat", - "valueFontSize": "150%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], - "description": "Process Resident Memory Usage", - "format": "decbytes", - "gauge": { - "maxValue": 1000, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 19, - "y": 1 - }, - "id": 24, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "150%", - "prefix": "", - "prefixFontSize": "150%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "avg(process_resident_memory_bytes{instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Resident Memory Usage", - "type": "singlestat", - "valueFontSize": "150%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "Average user and system CPU time spent in seconds.", - "fill": 1, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 4 - }, - "id": 30, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(rate(process_cpu_seconds_total{instance=\"$instance\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "CPU Time", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Average CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "Virtual and Resident memory size in bytes, averages over 5 min interval", - "fill": 1, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 4 - }, - "id": 32, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(rate(process_resident_memory_bytes{instance=\"$instance\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Resident Mem", - "refId": "A" - }, - { - "expr": "avg(rate(process_virtual_memory_bytes{instance=\"$instance\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Virtual Mem", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Average Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "Number of open file descriptors", - "fill": 1, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 4 - }, - "id": 34, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_open_fds{instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Open FD", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Open File Descriptors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 20, - "panels": [], - "title": "Registry Metrics", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "The HTTP requests", - "fill": 1, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 12 - }, - "id": 26, - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 3, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "ceil(rate(registry_http_requests_total{instance=\"$instance\"}[5m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ handler }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "HTTP Requests", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "The in-flight HTTP requests", - "fill": 1, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 12 - }, - "id": 18, - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "registry_http_in_flight_requests{instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ handler }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "In-Flight HTTP Requests", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "The rate of registry cache requests", - "fill": 1, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 12 - }, - "id": 28, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 3, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "ceil(rate(registry_storage_cache_total{instance=\"$instance\"}[5m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ type }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Registry Cache Requests Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fill": 1, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 20 - }, - "id": 44, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "increase(registry_storage_action_seconds_sum{instance=\"$instance\"}[2m]) * 1000", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{ action }}", - "refId": "A", - "step": 10, - "target": "" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Registry Action Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "timeseries", - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 20 - }, - "heatmap": {}, - "highlightCards": true, - "id": 36, - "legend": { - "show": false - }, - "links": [], - "targets": [ - { - "expr": "rate(registry_http_request_duration_seconds_bucket{handler=\"blob\"}[10m])", - "format": "heatmap", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "HTTP Request Latencies in seconds (blob)", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "short", - "logBase": 1, - "max": null, - "min": "0", - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "timeseries", - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 20 - }, - "heatmap": {}, - "highlightCards": true, - "id": 38, - "legend": { - "show": false - }, - "links": [], - "targets": [ - { - "expr": "rate(registry_http_request_duration_seconds_bucket{handler=\"base\"}[10m])", - "format": "heatmap", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Upload HTTP Request Latencies in seconds (base)", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "short", - "logBase": 1, - "max": null, - "min": "0", - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "timeseries", - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 27 - }, - "heatmap": {}, - "highlightCards": true, - "id": 2, - "legend": { - "show": false - }, - "links": [], - "targets": [ - { - "expr": "rate(registry_http_request_duration_seconds_bucket{handler=\"blob_upload\"}[10m])", - "format": "heatmap", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Upload HTTP Request Latencies in seconds (blob_upload)", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "short", - "logBase": 1, - "max": null, - "min": "0", - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "timeseries", - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 27 - }, - "heatmap": {}, - "highlightCards": true, - "id": 42, - "legend": { - "show": false - }, - "links": [], - "targets": [ - { - "expr": "rate(registry_http_request_duration_seconds_bucket{handler=\"manifest\"}[10m])", - "format": "heatmap", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Manifest HTTP Request Latencies in seconds (manifest)", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "short", - "logBase": 1, - "max": null, - "min": "0", - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "timeseries", - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 27 - }, - "heatmap": {}, - "highlightCards": true, - "id": 40, - "legend": { - "show": false - }, - "links": [], - "targets": [ - { - "expr": "rate(registry_http_request_duration_seconds_bucket{handler=\"catalog\"}[10m])", - "format": "heatmap", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Catalog HTTP Request Latencies in seconds (catalog)", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "short", - "logBase": 1, - "max": null, - "min": "0", - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - } - ], - "refresh": "10s", - "schemaVersion": 16, - "style": "dark", - "tags": ["docker", "registry", "internals"], - "templating": { - "list": [ - { - "allValue": null, - "current": {}, - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "namespace", - "options": [], - "query": "query_result(registry_http_in_flight_requests)", - "refresh": 2, - "regex": "/.*kubernetes_namespace=\"([^\"]+).*/", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "app", - "options": [], - "query": "query_result(registry_http_in_flight_requests{kubernetes_namespace=\"$namespace\"})", - "refresh": 2, - "regex": "/.*app=\"([^\"]+)/", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "instance", - "options": [], - "query": "query_result(up{app=\"$app\"})", - "refresh": 1, - "regex": "/.*instance=\"([^\"]+).*/", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] - }, - "timezone": "", - "title": "Docker Registry", - "uid": "CoBSgj8iz", - "version": 19 - }, - "overwrite": true -} diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json b/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json deleted file mode 100644 index c6df255..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_storage.json +++ /dev/null @@ -1,3261 +0,0 @@ -{ - "dashboard": { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "MinIO Grafana Dashboard - https://min.io/", - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 13502, - "graphTooltip": 0, - "id": null, - "links": [ - { - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": ["minio"], - "type": "dashboards" - } - ], - "liveNow": false, - "panels": [ - { - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "dtdurations", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 0, - "y": 0 - }, - "id": 1, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "exemplar": true, - "expr": "time() - max(minio_node_process_starttime_seconds{job=~\"$scrape_jobs\"})", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Uptime", - "type": "stat" - }, - { - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 3, - "y": 0 - }, - "id": 65, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "exemplar": true, - "expr": "sum by (instance) (minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"})", - "format": "table", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Total S3 Ingress", - "type": "stat" - }, - { - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "unit": "bytes", - "unitScale": true - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Used" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 6, - "y": 0 - }, - "id": 50, - "interval": "1m", - "links": [], - "maxDataPoints": 100, - "options": { - "displayLabels": [], - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "values": ["percent"] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.2.1", - "targets": [ - { - "exemplar": true, - "expr": "topk(1, sum(minio_cluster_capacity_usable_total_bytes{job=~\"$scrape_jobs\"}) by (instance)) - topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=~\"$scrape_jobs\"}) by (instance))", - "format": "time_series", - "instant": false, - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "Used", - "refId": "A", - "step": 300 - }, - { - "exemplar": true, - "expr": "topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=~\"$scrape_jobs\"}) by (instance)) ", - "hide": false, - "interval": "1m", - "legendFormat": "Free", - "refId": "B" - } - ], - "title": "Capacity", - "type": "piechart" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes", - "unitScale": true - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Objects" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": ["Usage"], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 10, - "y": 0 - }, - "id": 68, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.2.1", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "max(minio_cluster_usage_total_bytes{job=~\"$scrape_jobs\"})", - "interval": "", - "legendFormat": "Usage", - "range": true, - "refId": "A" - } - ], - "title": "Data Usage Growth", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-GrYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 5, - "x": 16, - "y": 0 - }, - "id": 52, - "links": [], - "options": { - "displayMode": "lcd", - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "code", - "exemplar": true, - "expr": "minio_cluster_objects_size_distribution{job=~\"$scrape_jobs\"}", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{range}}", - "refId": "A", - "step": 300, - "useBackend": false - } - ], - "title": "Object Size Distribution", - "type": "bargauge" - }, - { - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 21, - "y": 0 - }, - "id": 61, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{server}}", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Open FDs ", - "type": "timeseries" - }, - { - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 3, - "y": 3 - }, - "id": 64, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "exemplar": true, - "expr": "sum by (instance) (minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"})", - "format": "table", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Total S3 Egress", - "type": "stat" - }, - { - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 21, - "y": 3 - }, - "id": 62, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "minio_node_go_routine_total{job=~\"$scrape_jobs\"}", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{server}}", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Goroutines", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bool_on_off", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 0, - "y": 6 - }, - "id": 94, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "code", - "exemplar": true, - "expr": "minio_cluster_health_status{job=~\"$scrape_jobs\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "interval": "", - "legendFormat": "Pool: {{pool}} Set: {{set}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Cluster Health Status", - "transformations": [], - "type": "stat" - }, - { - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 6 - }, - "id": 78, - "links": [], - "maxDataPoints": 100, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "editorMode": "code", - "exemplar": false, - "expr": "max(minio_cluster_drive_online_total{job=~\"$scrape_jobs\"})", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": ".", - "metric": "process_start_time_seconds", - "range": false, - "refId": "A", - "step": 60 - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "max(minio_cluster_drive_offline_total{job=~\"$scrape_jobs\"})", - "format": "time_series", - "hide": false, - "instant": true, - "legendFormat": ".", - "range": false, - "refId": "B" - } - ], - "title": "Total Online/Offline Drives", - "type": "gauge" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "dark-yellow", - "value": 75000000 - }, - { - "color": "dark-red", - "value": 100000000 - } - ] - }, - "unit": "short", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 6, - "y": 6 - }, - "id": 66, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "max(minio_cluster_bucket_total{job=~\"$scrape_jobs\"})", - "format": "time_series", - "instant": false, - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Number of Buckets", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 7, - "x": 9, - "y": 6 - }, - "id": 63, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "exemplar": true, - "expr": "sum by (server) (rate(minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "Data Received [{{server}}]", - "refId": "A" - } - ], - "title": "S3 API Ingress Rate ", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 6 - }, - "id": 70, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "exemplar": true, - "expr": "sum by (server) (rate(minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "Data Sent [{{server}}]", - "refId": "A" - } - ], - "title": "S3 API Egress Rate ", - "type": "timeseries" - }, - { - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 0, - "y": 8 - }, - "id": 53, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "exemplar": true, - "expr": "max(minio_cluster_nodes_online_total{job=~\"$scrape_jobs\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Total Online Servers", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "dark-yellow", - "value": 75000000 - }, - { - "color": "dark-red", - "value": 100000000 - } - ] - }, - "unit": "short", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 6, - "y": 9 - }, - "id": 44, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "max(minio_cluster_usage_object_total{job=~\"$scrape_jobs\"})", - "format": "time_series", - "instant": false, - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Number of Objects", - "type": "stat" - }, - { - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ns", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 0, - "y": 10 - }, - "id": 80, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "exemplar": true, - "expr": "max(minio_heal_time_last_activity_nano_seconds{job=~\"$scrape_jobs\"})", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Time Since Last Heal", - "type": "stat" - }, - { - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ns", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 3, - "y": 10 - }, - "id": 81, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "exemplar": true, - "expr": "max(minio_usage_last_activity_nano_seconds{job=~\"$scrape_jobs\"})", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Time Since Last Scan", - "type": "stat" - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 9, - "x": 0, - "y": 12 - }, - "hiddenSeries": false, - "id": 60, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (server,api) (increase(minio_s3_requests_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{server,api}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 API Request Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 7, - "x": 9, - "y": 12 - }, - "hiddenSeries": false, - "id": 88, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (server,api) (increase(minio_s3_requests_4xx_errors_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{server,api}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 API Request Error Rate (4xx)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 12 - }, - "hiddenSeries": false, - "id": 86, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (server,api) (increase(minio_s3_requests_5xx_errors_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{server,api}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 API Request Error Rate (5xx)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 99, - "options": { - "displayMode": "lcd", - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "minio_cluster_health_erasure_set_online_drives{job=~\"$scrape_jobs\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Pool {{pool}} / Set {{set}} - Online Drives", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "minio_cluster_health_erasure_set_read_quorum{job=~\"$scrape_jobs\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Pool {{pool}} / Set {{set}} - Read Quorum", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "minio_cluster_health_erasure_set_write_quorum{job=~\"$scrape_jobs\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Pool {{pool}} / Set {{set}} - Write Quorum", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "minio_cluster_health_erasure_set_healing_drives{job=~\"$scrape_jobs\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Pool {{pool}} / Set {{set}} - Healing Drives", - "range": true, - "refId": "D", - "useBackend": false - }, - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "minio_cluster_health_erasure_set_status{job=~\"$scrape_jobs\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Pool {{pool}} / Set {{set}} - Status", - "range": true, - "refId": "E", - "useBackend": false - } - ], - "title": "Health Breakdown", - "type": "bargauge" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-GrYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 76, - "options": { - "displayMode": "lcd", - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "editorMode": "code", - "exemplar": false, - "expr": "minio_node_process_resident_memory_bytes{job=~\"$scrape_jobs\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{server}}", - "range": true, - "refId": "A" - } - ], - "title": "Memory Usage ", - "type": "bargauge" - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 26 - }, - "hiddenSeries": false, - "id": 73, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "rate(minio_node_io_rchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Node RChar [{{server}}]", - "refId": "A" - }, - { - "exemplar": true, - "expr": "rate(minio_node_io_wchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "", - "legendFormat": "Node WChar [{{server}}]", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Read, Write I/O", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:381", - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:382", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green" - }, - { - "color": "orange", - "value": 70 - }, - { - "color": "red", - "value": 85 - } - ] - }, - "unit": "s", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 26 - }, - "id": 77, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "rate(minio_node_process_cpu_total_seconds{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{server}}", - "range": true, - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "description": "Total number of bytes received and sent on MinIO cluster", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 17, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "rate(minio_inter_node_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Internode Bytes Received [{{server}}]", - "metric": "minio_http_requests_duration_seconds_count", - "range": true, - "refId": "A", - "step": 4 - }, - { - "exemplar": true, - "expr": "rate(minio_inter_node_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "", - "legendFormat": "Internode Bytes Sent [{{server}}]", - "refId": "B" - } - ], - "title": "Internode Traffic", - "type": "timeseries" - }, - { - "aliasColors": { - "available 10.13.1.25:9000": "green", - "used 10.13.1.25:9000": "blue" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "", - "fieldConfig": { - "defaults": { - "links": [], - "unit": "bytes", - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 33 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", - "interval": "", - "legendFormat": "Open FDs [{{server}}]", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "File Descriptors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "Offline 10.13.1.25:9000": "dark-red", - "Total 10.13.1.25:9000": "blue" - }, - "bars": true, - "dashLength": 10, - "dashes": false, - "description": "Number of online drives per MinIO Server", - "fieldConfig": { - "defaults": { - "links": [], - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 40 - }, - "hiddenSeries": false, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "rate(minio_node_syscall_read_total{job=~\"$scrape_jobs\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Read Syscalls [{{server}}]", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - }, - { - "exemplar": true, - "expr": "rate(minio_node_syscall_write_total{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "", - "legendFormat": "Write Syscalls [{{server}}]", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Syscalls", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:185", - "decimals": 0, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:186", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 40 - }, - "hiddenSeries": false, - "id": 95, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "rate(minio_node_scanner_objects_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "1m", - "legendFormat": "[{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scanned Objects", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 47 - }, - "hiddenSeries": false, - "id": 75, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "rate(minio_node_scanner_versions_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "1m", - "legendFormat": "[{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scanned Versions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 47 - }, - "hiddenSeries": false, - "id": 96, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "rate(minio_node_scanner_directories_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "1m", - "legendFormat": "[{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Scanned Directories", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "dtdurations", - "unitScale": true - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 54 - }, - "id": 89, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.3.1", - "targets": [ - { - "exemplar": true, - "expr": "minio_cluster_kms_uptime{job=~\"$scrape_jobs\"}", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "metric": "minio_cluster_kms_uptime", - "refId": "A", - "step": 60 - } - ], - "title": "KMS Uptime", - "type": "stat" - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 54 - }, - "hiddenSeries": false, - "id": 91, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (server) (increase(minio_cluster_kms_request_error{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{server}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "KMS Request 4xx Error Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fieldConfig": { - "defaults": { - "unit": "bool_on_off", - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 54 - }, - "hiddenSeries": false, - "id": 90, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (server) (minio_cluster_kms_online{job=~\"$scrape_jobs\"})", - "interval": "1m", - "legendFormat": "{{server}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "KMS Online(1)/Offline(0)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "bool_on_off", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 54 - }, - "hiddenSeries": false, - "id": 98, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "rate(minio_node_scanner_bucket_scans_finished{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "1m", - "legendFormat": "[{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Bucket Scans Finished", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 58 - }, - "hiddenSeries": false, - "id": 92, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (server) (increase(minio_cluster_kms_request_failure{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{server}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "KMS Request 5xx Error Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 58 - }, - "hiddenSeries": false, - "id": 93, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (server) (rate(minio_cluster_kms_request_success{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "KMS Request Success [{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "KMS Request Success Rate ", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fieldConfig": { - "defaults": { - "unitScale": true - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 61 - }, - "hiddenSeries": false, - "id": 97, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.3.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "rate(minio_node_scanner_bucket_scans_started{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "1m", - "legendFormat": "[{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Bucket Scans Started", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "refresh": "", - "schemaVersion": 39, - "tags": ["minio"], - "templating": { - "list": [ - { - "current": {}, - "definition": "label_values(job)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "scrape_jobs", - "options": [], - "query": { - "query": "label_values(job)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] - }, - "timezone": "", - "title": "MinIO Dashboard", - "uid": "TgmJnqnnk", - "version": 54, - "weekStart": "" - }, - "overwrite": true -} diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_valkey.json b/rootfs/usr/share/grafana/api/dashboards/drycc_valkey.json deleted file mode 100644 index a712c13..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_valkey.json +++ /dev/null @@ -1,1367 +0,0 @@ -{ - "dashboard": { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Redis Dashboard for Prometheus Redis Exporter 1.x", - "editable": true, - "gnetId": 763, - "graphTooltip": 1, - "id": null, - "iteration": 1602758020790, - "links": [], - "panels": [ - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 24, - "panels": [], - "title": "Performance", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Average taken across instances", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 1 - }, - "hiddenSeries": false, - "id": 18, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "avg(irate(redis_commands_total{instance=~\"$instance\"} [$__rate_interval])) by (cmd)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{cmd}}", - "metric": "redis_command_calls_total", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Commands per second", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Average taken across instances", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 1 - }, - "hiddenSeries": false, - "id": 20, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(irate(redis_commands_duration_seconds_total{instance=~\"$instance\"}[$__rate_interval])) by (cmd)\n /\navg(irate(redis_commands_total{instance=~\"$instance\"}[$__rate_interval])) by (cmd)\n", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ cmd }}", - "metric": "redis_command_calls_total", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Command latency per second", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "Hit ratio": "blue" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "decimals": 2, - "description": "Hit rate shows the percentage of key space lookups that hit a key.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 1 - }, - "hiddenSeries": false, - "id": 1, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "dataLinks": [] - }, - "percentage": true, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/Target/", - "color": "#56A64B", - "dashes": true, - "fill": 0, - "hideTooltip": true, - "linewidth": 1 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(irate(redis_keyspace_hits_total{instance=~\"$instance\"}[$__rate_interval]) / (irate(redis_keyspace_misses_total{instance=~\"$instance\"}[$__rate_interval]) + irate(redis_keyspace_hits_total{instance=~\"$instance\"}[$__rate_interval]))) by (instance)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "", - "refId": "A", - "step": 240, - "target": "" - }, - { - "expr": "1", - "interval": "", - "legendFormat": "Target hit ratio for cache", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Hit ratio per instance", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": "", - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 22, - "panels": [], - "title": "Memory", - "type": "row" - }, - { - "aliasColors": { - "max": "#BF1B00" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Total taken across instances", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 9 - }, - "hiddenSeries": false, - "id": 7, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/max/", - "color": "#E02F44", - "dashes": true, - "fill": 0, - "linewidth": 1 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(redis_memory_used_bytes{instance=~\"$instance\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Used Memory", - "metric": "", - "refId": "A", - "step": 240, - "target": "" - }, - { - "expr": "sum(redis_memory_max_bytes{instance=~\"$instance\"})", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "Configured max memory", - "refId": "B", - "step": 240 - }, - { - "expr": "sum(redis_memory_used_rss_bytes{instance=~\"$instance\"})", - "interval": "", - "legendFormat": "Used RSS memory", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Total Memory Usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "Recommend restart redis": "red" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 9 - }, - "hiddenSeries": false, - "id": 10, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/restart/", - "color": "#E02F44", - "dashes": true, - "fill": 0, - "linewidth": 1 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "redis_memory_fragmentation_ratio{instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory fragmentation ratio per instance", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "Evictions": "red", - "evicts": "#890F02", - "memcached_items_evicted_total{instance=\"172.17.0.1:9150\",job=\"prometheus\"}": "#890F02", - "reclaims": "#3F6833", - "{container=\"redis-exporter\", instance=\"redis-86cb5d76d7-fcdln:redis-exporter:redis-metrics\", job=\"default/redis\", namespace=\"default\", pod=\"redis-86cb5d76d7-fcdln\"}": "red", - "{instance=\"redis-86cb5d76d7-fcdln:redis-exporter:redis-metrics\"}": "red" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 9 - }, - "hiddenSeries": false, - "id": 8, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "reclaims", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(redis_evicted_keys_total{instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "B", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Key evictions per second per instance", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 26, - "panels": [], - "title": "Basic activity", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Sum taken across instances", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 17 - }, - "hiddenSeries": false, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(redis_connected_clients{instance=~\"$instance\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Connected", - "refId": "A" - }, - { - "expr": "sum(redis_blocked_clients{instance=~\"$instance\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Blocked", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Connected/Blocked Clients", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "db1": "yellow" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Sum taken across instances", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 17 - }, - "hiddenSeries": false, - "id": 5, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum (redis_db_keys{instance=~\"$instance\"}) by (db)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ db }}", - "refId": "A", - "step": 240, - "target": "" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Total Items per DB", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Sum taken across instances", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 17 - }, - "hiddenSeries": false, - "id": 13, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum (redis_db_keys{instance=~\"$instance\"}) - sum (redis_db_keys_expiring{instance=~\"$instance\"})", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "Not expiring", - "refId": "A", - "step": 240, - "target": "" - }, - { - "expr": "sum(redis_db_keys_expiring{instance=~\"$instance\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Expiring", - "metric": "", - "refId": "B", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Expiring vs Not-Expiring Keys", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "This metric will only be non-zero if the instance is a master", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 24 - }, - "hiddenSeries": false, - "id": 28, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(redis_connected_slaves{instance=~\"$instance\"}) by (instance)", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Connected slaves by instance", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "This metric is only exported if the instance is a slave.", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 24 - }, - "hiddenSeries": false, - "id": 30, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "redis_master_last_io_seconds_ago{instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Time since last master connection", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": false, - "schemaVersion": 25, - "style": "dark", - "tags": ["prometheus", "redis"], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": true, - "tags": [], - "text": "redisdb-7d6b98cd98-kjt5x + redisdb-7d6b98cd98-pbkg6", - "value": ["redisdb-7d6b98cd98-kjt5x", "redisdb-7d6b98cd98-pbkg6"] - }, - "datasource": "${datasource}", - "definition": "label_values(redis_up, instance)", - "hide": 0, - "includeAll": false, - "label": null, - "multi": true, - "name": "instance", - "options": [], - "query": "label_values(redis_up, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "prometheus", - "value": "prometheus" - }, - "hide": 0, - "includeAll": false, - "label": "Data Source", - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - } - ] - }, - "time": { - "from": "now-12h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] - }, - "timezone": "browser", - "title": "Redis Dashboard for Prometheus Redis Exporter 1.x", - "uid": "bRd48yKMdd", - "version": 5 - }, - "overwrite": true -} diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_cluster.json b/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_cluster.json deleted file mode 100644 index a6e3d6e..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_cluster.json +++ /dev/null @@ -1,9923 +0,0 @@ -{ - "dashboard": { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "enable": true, - "expr": "sum(ALERTS{job=~\"$job\", instance=~\"$instance\", alertgroup=\"vmcluster\",alertstate=\"firing\",show_at=\"dashboard\"}) by(alertname)", - "hide": false, - "iconColor": "red", - "name": "alerts", - "titleFormat": "{{alertname}}" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "enable": true, - "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(version))", - "hide": true, - "iconColor": "dark-blue", - "name": "version change", - "textFormat": "{{version}}", - "titleFormat": "Version change" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "enable": true, - "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job)", - "hide": false, - "iconColor": "dark-yellow", - "name": "restarts", - "textFormat": "{{job}} restarted" - } - ] - }, - "description": "Overview for cluster VictoriaMetrics v1.102.0 or higher", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 1, - "links": [ - { - "icon": "doc", - "tags": [], - "targetBlank": true, - "title": "Cluster Wiki", - "type": "link", - "url": "https://docs.victoriametrics.com/cluster-victoriametrics" - }, - { - "icon": "external link", - "tags": [], - "targetBlank": true, - "title": "Found a bug?", - "type": "link", - "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues" - }, - { - "icon": "external link", - "tags": [], - "targetBlank": true, - "title": "New releases", - "type": "link", - "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases" - } - ], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 137, - "panels": [], - "title": "Stats", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "How many [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) are in storage", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 1 - }, - "id": 131, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "exemplar": true, - "expr": "sum(vm_rows{job=~\"$job_storage\", type!~\"indexdb.*\"})", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total datapoints", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the datapoints ingestion rate, including replication factor.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 1 - }, - "id": 124, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(vm_vminsert_metrics_read_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) ", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Ingestion rate", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the rate of HTTP read requests.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "req/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 12, - "y": 1 - }, - "id": 130, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vm_http_requests_total{job=~\"$job\", instance=~\"$instance\", path=~\"/select/.*\"}[$__rate_interval]))", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Read requests", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Total number of available CPUs for all VM components. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 18, - "y": 1 - }, - "id": 126, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"})", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Available CPU", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series) with new data points inserted during the last hour. High value may result in ingestion slowdown.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 4 - }, - "id": 34, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(max_over_time(vm_cache_entries{job=~\"$job\", instance=~\"$instance\", type=\"storage/hour_metric_ids\"}[1h]))", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Active series", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Total amount of used disk space", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 4 - }, - "id": 35, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "exemplar": false, - "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\"})", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Disk space usage", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Average disk usage per datapoint.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 12, - "y": 4 - }, - "id": 112, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "exemplar": true, - "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\"}) / sum(vm_rows{job=~\"$job_storage\", type!~\"indexdb.*\"})", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Bytes per point", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Total size of available memory for all VM components.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 18, - "y": 4 - }, - "id": 128, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"})", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Available memory", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "minWidth": 50 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "displayName", - "value": "Count" - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 7 - }, - "id": 149, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Count" - } - ] - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)", - "format": "table", - "instant": true, - "range": false, - "refId": "A" - } - ], - "title": "", - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepAfter", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 16, - "x": 8, - "y": 7 - }, - "id": 62, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(min_over_time(up{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)", - "format": "time_series", - "instant": false, - "legendFormat": "{{job}}", - "refId": "A" - } - ], - "title": "Uptime ($job)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 10, - "panels": [], - "title": "Overview", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "How many [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) are inserted into cluster per second by protocol before the replication. Check vminsert metrics if there are any issues with ingestion.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 13 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vm_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (type) > 0 ", - "interval": "", - "legendFormat": "{{type}}", - "range": true, - "refId": "A" - } - ], - "title": "Datapoints ingestion rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 13 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{path}}", - "range": true, - "refId": "A" - } - ], - "title": "Requests rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series) with new data points inserted during the last hour across all storage nodes. High value may result in ingestion slowdown and high memory usage.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 21 - }, - "id": 12, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "sum(vm_cache_entries{job=~\"$job\", instance=~\"$instance\", type=\"storage/hour_metric_ids\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Active time series", - "refId": "A" - } - ], - "title": "Active time series ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "The less time it takes is better.\n* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 21 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(vm_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", quantile=\"0.99\"}) by (path) > 0", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{path}}", - "range": true, - "refId": "A" - } - ], - "title": "Query duration 0.99 quantile ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 29 - }, - "id": 52, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, path) > 0", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{path}} ({{job}})", - "range": true, - "refId": "A" - } - ], - "title": "Requests error rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/oS7Bi_0Wz?viewPanel=203&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 29 - }, - "id": 104, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, level) > 0", - "format": "time_series", - "hide": false, - "interval": "5m", - "intervalFactor": 1, - "legendFormat": "{{job}} - {{level}}", - "range": true, - "refId": "A" - } - ], - "title": "Logging rate", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 37 - }, - "id": 46, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Percentage of used RSS memory (resident).\nThe RSS memory shows the amount of memory recently accessed by the application. It includes anonymous memory and data from recently accessed files (aka page cache).\nThe application's performance will significantly degrade when memory usage is close to 100%.\n\nClick on the line and choose Drilldown to show memory usage per instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 38 - }, - "id": 66, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "RSS memory % usage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%\n\nClick on the line and choose Drilldown to show memory usage per instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/oS7Bi_0Wz?viewPanel=190&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 38 - }, - "id": 138, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "RSS anonymous memory % usage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 160 - }, - "id": 64, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "CPU ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of bytes read/write from the storage layer.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/read .*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 160 - }, - "id": 122, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "read {{job}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "write {{job}}", - "range": true, - "refId": "B" - } - ], - "title": "Disk writes/reads ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the percentage of open file descriptors compared to the limit set in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/max.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C4162A", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 168 - }, - "id": 117, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n max_over_time(process_open_fds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_max_fds{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{job}}", - "range": true, - "refId": "A" - } - ], - "title": "Open FDs usage % ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of read/write syscalls such as read, pread, write, pwrite.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/read .*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 168 - }, - "id": 204, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(process_io_read_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "read {{job}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(process_io_write_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "write {{job}}", - "range": true, - "refId": "B" - } - ], - "title": "Disk write/read calls ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 176 - }, - "id": 68, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(max_over_time(go_goroutines{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Goroutines ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 176 - }, - "id": 119, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(max_over_time(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job)", - "interval": "", - "legendFormat": "{{job}}", - "range": true, - "refId": "A" - } - ], - "title": "TCP connections ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 184 - }, - "id": 70, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(max_over_time(process_num_threads{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Threads ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 184 - }, - "id": 120, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "TCP connections rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/troubleshooting/ ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 192 - }, - "id": 210, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(\n rate(go_gc_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n ) by(job)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "CPU spent on GC ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.1 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 192 - }, - "id": 213, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, instance, le))) by(job)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Go scheduling latency", - "type": "timeseries" - } - ], - "title": "Resource usage ($job)", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 38 - }, - "id": 106, - "panels": [ - { - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 39 - }, - "id": 211, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "See [Troubleshooting](https://docs.victoriametrics.com/troubleshooting/) docs.", - "mode": "markdown" - }, - "pluginVersion": "11.5.0", - "title": "", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the rate and total number of new series created over last 24h.\n\nHigh [churn rate](https://docs.victoriametrics.com/faq/#what-is-high-churn-rate) tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected [cardinality](https://docs.victoriametrics.com/keyconcepts/#cardinality) \"explosions\".\n\nThe higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.\n\nTo investigate stats about most expensive series use `api/v1/status/tsdb` handler. More details here https://docs.victoriametrics.com/cluster-victoriametrics/#url-format\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://valyala.medium.com/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "new series over 24h" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.axisSoftMin", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 104 - }, - "id": 102, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "exemplar": true, - "expr": "sum(rate(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "churn rate", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "exemplar": true, - "expr": "sum(increase(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[24h]))", - "hide": false, - "interval": "", - "legendFormat": "new series over 24h", - "refId": "B" - } - ], - "title": "Churn rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "The percentage of [slow inserts](https://docs.victoriametrics.com/faq/#what-is-a-slow-insert) compared to the total ingestion rate. \\n\\nThe lower the better. \\n\\nIn short, slow insert is a cache miss. There are following reasons for slow inserts to go up: \\n* Ingestion of completely new, not seen before time series;\\n* [Re-routing](https://docs.victoriametrics.com/cluster-victoriametrics/#cluster-availability) of series when one or more vmstorage nodes are unavailable;\\n* Not enough memory to maintain big enough caches for the current workload.\\n\\nIf percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series). \\n\\nSee [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "red", - "value": 0.1 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 104 - }, - "id": 108, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(\n rate(vm_slow_row_inserts_total{job=~\"$job_storage\"}[$__rate_interval]) \n / rate(vm_rows_added_to_storage_total{job=~\"$job_storage\"}[$__rate_interval])\n)", - "interval": "", - "legendFormat": "slow inserts", - "range": true, - "refId": "A" - } - ], - "title": "Slow inserts", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Merge assist happens when vmstorage can't keep up with merging parts. This is usually a sign of overload for vmstorage.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 112 - }, - "id": 170, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(increase(vm_assisted_merges_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type) > 0", - "format": "time_series", - "interval": "5m", - "intervalFactor": 1, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Assisted merges ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows % of slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.\n\nThe less value is better.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 112 - }, - "id": 107, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_slow_queries_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval]))\n/\nsum(rate(vm_http_requests_total{job=~\"$job_select\", instance=~\"$instance\", path=~\"/select/.*\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "slow queries %", - "range": true, - "refId": "A" - } - ], - "title": "Slow queries % ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 120 - }, - "id": 144, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"} \n /\n vm_cache_size_max_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(type)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Cache usage % by type ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows cache miss ratio. Lower is better.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 120 - }, - "id": 58, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n rate(vm_cache_misses_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n rate(vm_cache_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n) by(type)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Cache miss ratio ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "The value is above 0 when the vmstorage at the given `addr` communicates to the given vminsert node that it cannot accept new data because it is in the read-only mode.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 129 - }, - "id": 142, - "links": [ - { - "targetBlank": true, - "title": "Readonly mode", - "url": "https://docs.victoriametrics.com/cluster-victoriametrics/#readonly-mode" - } - ], - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(vm_rpc_vmstorage_is_read_only{job=~\"$job_insert\", instance=~\"$instance\"}) by(instance, addr) > 0", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}} => {{addr}}", - "range": true, - "refId": "A" - } - ], - "title": "Storage in readonly status for vminsert ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows rate of deduplicated samples during [deduplication](https://docs.victoriametrics.com/#deduplication) or [downsampling](https://docs.victoriametrics.com/#downsampling), according to the configured `-dedup.minScrapeInterval` or `-downsampling.period`.\n\nShould be empty if `-dedup.minScrapeInterval` and `-downsampling.period` isn't set.\n\nDeduplication happens in two places:\n* `type=\"merge\"` - during [background merges](https://docs.victoriametrics.com/#storage) by vmstorages. Deduplication during merges is permanent.\n* `type=\"select\"` - during [read queries](https://docs.victoriametrics.com/keyconcepts/#query-data) by vmselects. Deduplication happens in-flight and has no permanent effect.\n\nThe more samples need to be deduplicated, the higher will be resource usage on vmselects or vmstorages.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 129 - }, - "id": 215, - "links": [], - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vm_deduplicated_samples_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) >0", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Deduplication rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows how many samples were ignored or dropped on insertion due to various reasons:\n* timestamp out of retention period or timestamp in future;\n* invalid metric name;\n* exceeding limit for labels length or number;\n* dropped by [relabeling configuration](https://docs.victoriametrics.com/#relabeling).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 137 - }, - "id": 135, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(increase(vm_rows_ignored_total{job=~\"$job\", instance=~\"$instance\"}[1h])) by (reason)", - "hide": false, - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(increase(vm_relabel_metrics_dropped_total{job=~\"$job\", instance=~\"$instance\"}[1h]))", - "hide": false, - "instant": false, - "legendFormat": "relabeling", - "range": true, - "refId": "B" - } - ], - "title": "Samples dropped for last 1h ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of partial results served by `vmselects`.\n\n`vmselect` continues serving queries if at least a single vmstorage nodes is available. It marks responses as `partial` for queries served from the remaining healthy vmstorage nodes.\n\nIf you prefer consistency over availability then run vmselect nodes with `-search.denyPartialResponse` cmd-line flag. In this case vmselect returns an error if at least a single vmstorage node is unavailable.\n\nSee more at [cluster availability](https://docs.victoriametrics.com/cluster-victoriametrics/#cluster-availability).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepAfter", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 137 - }, - "id": 217, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(increase(vm_partial_results_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0", - "format": "time_series", - "instant": false, - "legendFormat": "partial results", - "refId": "A" - } - ], - "title": "Partial query results ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 145 - }, - "id": 183, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "job" - } - ] - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(flag{is_set=\"true\", job=~\"$job\", instance=~\"$instance\"}) by(job, instance, name, value)", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Non-default flags", - "transformations": [ - { - "id": "groupBy", - "options": { - "fields": { - "instance": { - "aggregations": ["uniqueValues"], - "operation": "aggregate" - }, - "job": { - "aggregations": [], - "operation": "groupby" - }, - "name": { - "aggregations": [], - "operation": "groupby" - }, - "value": { - "aggregations": [], - "operation": "groupby" - } - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepAfter", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 145 - }, - "id": 214, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", - "format": "time_series", - "instant": false, - "legendFormat": "{{job}}", - "refId": "A" - } - ], - "title": "Restarts ($job)", - "type": "timeseries" - } - ], - "title": "Troubleshooting", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 39 - }, - "id": 48, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows number of pushed and sent rows. \n* `Pushed rows` - rows added to internal inserter buffers before send\n* `Sent rows` - successfully transmitted rows to storage nodes\n\nPlease note, it could be that `Sent > Pushed` because of the replication factor.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1331 - }, - "id": 76, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_rpc_rows_pushed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pushed", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_rpc_rows_sent_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Sent", - "range": true, - "refId": "E" - } - ], - "title": "Rows ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Rate of RPC errors by type:\n* `Connection` - the number of connection errors to vmstorage node\n* `Dial` - the number of dial errors to vmstorage node.\n* `Handshake` - the number of handshake errors to vmstorage node\n* `Rerouted` - errors appeared during rerouting of rows from un-healthy storage node to a healthy one.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1331 - }, - "id": 86, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_rpc_connection_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Connection", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "sum(rate(vm_rpc_dial_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Dial", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "sum(rate(vm_rpc_handshake_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Handshake", - "refId": "E" - } - ], - "title": "RPC errors ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "The number of rows rerouted to the vmstorage node from other nodes when they were unhealthy.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "rps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1340 - }, - "id": 80, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vm_rpc_rows_rerouted_to_here_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr) > 0", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Rows ($instance) rerouted to ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "The number of rows rerouted from the vmstorage node to healthy nodes when the given node was unhealthy.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "rps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1340 - }, - "id": 78, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "exemplar": true, - "expr": "sum(rate(vm_rpc_rows_rerouted_from_here_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr) > 0", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{addr}}", - "refId": "A" - } - ], - "title": "Rows ($instance) rerouted from", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "The number of rows or bytes that vminsert internal buffer contains at the moment.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "bytes" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1348 - }, - "id": 82, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "sum(vm_rpc_buf_pending_bytes{job=~\"$job\", instance=~\"$instance\"})", - "legendFormat": "bytes", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "sum(vm_rpc_rows_pending{job=~\"$job\", instance=~\"$instance\"})", - "legendFormat": "rows", - "refId": "B" - } - ], - "title": "Pending", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Network usage by internal VictoriaMetrics RPC protocol", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1348 - }, - "id": 74, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) * 8", - "legendFormat": "network usage", - "range": true, - "refId": "A" - } - ], - "title": "RPC network usage ($instance)", - "type": "timeseries" - } - ], - "title": "Interconnection ($job)", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 24, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of datapoints ingested into storage nodes per second. This metric doesn't show all stored datapoints since some of them may be dropped because of wrong timestamps or decode errors.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 41 - }, - "id": 100, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_vminsert_metrics_read_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) ", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "ingestion rate", - "range": true, - "refId": "A" - } - ], - "title": "Ingestion rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the approx time needed to reach 100% of disk capacity for at least one vmstorage node based on the following params:\n* free disk space (after -storage.minFreeDiskSpaceBytes);\n* row ingestion rate;\n* compression.\n\nNote: this panel doesn't account for deduplication process.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/oS7Bi_0Wz?viewPanel=196&${__url_time_range}&${__all_variables}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 41 - }, - "id": 113, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "min"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "min((vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"} - vm_free_disk_space_limit_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) \n/ \nignoring(path) (\n rate(vm_rows_added_to_storage_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d])\n * scalar(\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n / \n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n )\n))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "min ETA", - "range": true, - "refId": "A" - } - ], - "title": "Storage full ETA ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=$job_storage&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 151, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"}\n)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "max", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "min(\n rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"}\n)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "min", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "quantile(0.5,\n rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"}\n)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "median", - "range": true, - "refId": "C" - } - ], - "title": "CPU usage % ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Percentage of used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=$job_storage&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 167, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}\n)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "max", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "min(\n max_over_time(process_resident_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}\n)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "min", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "quantile(0.5,\n max_over_time(process_resident_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}\n)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "median", - "range": true, - "refId": "C" - } - ], - "title": "Memory usage % ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. The `max` is an internal limit and can't be changed. It is always equal to the number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "max" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C4162A", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 56 - }, - "id": 212, - "links": [ - { - "targetBlank": true, - "title": "Related discussion", - "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/632" - } - ], - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n max_over_time(vm_concurrent_insert_current{job=~\"$job_storage\", \n instance=~\"$instance\"}[$__rate_interval])\n)", - "interval": "", - "legendFormat": "current", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "min(vm_concurrent_insert_capacity{job=~\"$job_storage\", instance=~\"$instance\"})", - "hide": false, - "interval": "", - "legendFormat": "max", - "range": true, - "refId": "B" - } - ], - "title": "Concurrent flushes on disk ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of concurrently executed read requests, where:\n* `max` - equal to `-search.maxConcurrentRequest` cmd-line flag;\n* `current` - current number of concurrent select requests executed by storage.\n\nEach concurrent select request requires RAM, CPU and disk IO resources during processing. Too many concurrent requests could result into OOM exceptions.\n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row).\n\nSee more at https://docs.victoriametrics.com/cluster-victoriametrics/#resource-usage-limits", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "max" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C4162A", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 133, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n max_over_time(vm_vmselect_concurrent_requests_current{job=~\"$job_storage\", \n instance=~\"$instance\"}[$__rate_interval])\n)", - "interval": "", - "legendFormat": "current", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "min(vm_vmselect_concurrent_requests_capacity{job=~\"$job_storage\", instance=~\"$instance\"})", - "hide": false, - "interval": "", - "legendFormat": "max", - "range": true, - "refId": "B" - } - ], - "title": "Concurrent selects ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "The max number of on-going merges across storage nodes.\n The drastic change in number of merges could be a sign of on-going deduplication/downsampling activity.\n It is expected to have high numbers for `storage/small` metric.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 64 - }, - "id": 54, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(max_over_time(vm_active_merges{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type)", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Active merges ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "The number of rows merged per second by storage nodes. Merge speed depends on available CPU and disk IO bandwidth.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 64 - }, - "id": 55, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_rows_merged_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type)", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Merge speed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/oS7Bi_0Wz?viewPanel=200&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 72 - }, - "id": 20, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "max", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "min(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "min", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "quantile(0.5,\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "median", - "range": true, - "refId": "C" - } - ], - "title": "Disk space usage % ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "The max number of data parts of LSM tree across all storage nodes in the last [partition](https://docs.victoriametrics.com/#storage).\nIncrease in number of parts (the hard limit is 512) is an evidence of slow merge performance - check the resource utilization.\n* `indexdb` - inverted index\n* `storage/small` - recently added parts of data ingested into storage (hot data)\n* `storage/big` - small parts gradually merged into bigger parts (cold data)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 72 - }, - "id": 22, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(vm_parts{job=~\"$job_storage\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(type)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(vm_last_partition_parts{job=~\"$job_storage\", instance=~\"$instance\"}) by(type)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "range": true, - "refId": "B" - } - ], - "title": "LSM parts max by type ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the percentage of used disk space by type: datapoints or indexdb. Normally, indexdb takes much less space comparing to datapoints. But with high [churn rate](https://docs.victoriametrics.com/faq/#what-is-high-churn-rate) the size of the indexdb could grow significantly.\n\nThe sum of the % can be > 100% since panel shows max % per-job and per-instance. It means different instance can have different ratio between datapoints and indexdb size.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/oS7Bi_0Wz?viewPanel=201&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 80 - }, - "id": 202, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(job, instance)\n / \n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "indexdb", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"}) by(job, instance)\n / \n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "datapoints", - "range": true, - "refId": "B" - } - ], - "title": "Disk space usage % by type ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `3*`, since VictoriaMetrics pushes pending data to persistent storage every two seconds. The index datapoints value in general is much lower.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "pending index entries" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "decimals", - "value": 3 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 80 - }, - "id": 14, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(max_over_time(vm_pending_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"storage\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "pending datapoints", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(max_over_time(vm_pending_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "pending index entries", - "range": true, - "refId": "B" - } - ], - "title": "Pending datapoints ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows network usage by vmstorage services.\n* Writes show traffic sent to clients.\n* Reads show traffic received from clients.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 88 - }, - "id": 206, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(name) * 8 > 0", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "read from {{name}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(name) * 8 > 0", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "write to {{name}}", - "range": true, - "refId": "B" - } - ], - "title": "Network usage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Value > 0 means vmstorage is in readonly mode.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 88 - }, - "id": 141, - "links": [ - { - "targetBlank": true, - "title": "Readonly mode", - "url": "https://docs.victoriametrics.com/cluster-victoriametrics/#readonly-mode" - } - ], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "vm_storage_is_read_only{job=~\"$job_storage\", instance=~\"$instance\"} > 0", - "interval": "", - "legendFormat": "{{ instance }}", - "range": true, - "refId": "A" - } - ], - "title": "Readonly mode", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the max number of existing [snapshots](https://docs.victoriametrics.com/#how-to-work-with-snapshots) across vmstorages.\n\nRecently made snapshots do not occupy disk space. But with time, as snapshots become older, they start to occupy more and more disk space.\n\nIt is recommended deleting old snapshots when they are no longer needed to free up disk space.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 96 - }, - "id": 216, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(vm_snapshots{job=~\"$job_storage\", instance=~\"$instance\"})", - "interval": "", - "legendFormat": "snapshots", - "range": true, - "refId": "A" - } - ], - "title": "Number of snapshots", - "type": "timeseries" - } - ], - "title": "vmstorage ($instance)", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 41 - }, - "id": 42, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Request rate accepted by vmselect nodes", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1344 - }, - "id": 92, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_http_requests_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Requests rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the max number of concurrent selects across instances.\n* `max` limit can be configured via `search.maxConcurrentRequests` flag\n* `current` shows the current number of goroutines busy with processing requests\n\nWhen `current` hits `max` constantly, it means one or more vmselect nodes are overloaded with number of requests. If you observe that CPU for vmselects is saturated, consider adding more vmselect replicas or increase CPU resources. If CPU panel shows a plenty of free resources - try increasing `search.maxConcurrentRequests`.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "max" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C4162A", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1344 - }, - "id": 95, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(max_over_time(vm_concurrent_select_current{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) ", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "current", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "min(vm_concurrent_select_capacity{job=~\"$job_select\", instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "B" - } - ], - "title": "Concurrent selects ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=$job_select&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1352 - }, - "id": 163, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_select\", instance=~\"$instance\"}\n)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "max", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "min(\n rate(process_cpu_seconds_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_select\", instance=~\"$instance\"}\n)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "min", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "quantile(0.5,\n rate(process_cpu_seconds_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_select\", instance=~\"$instance\"}\n)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "median", - "range": true, - "refId": "C" - } - ], - "title": "CPU usage % ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=$job_select&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1352 - }, - "id": 165, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}\n)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "max", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "min(\n max_over_time(process_resident_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}\n)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "min", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "quantile(0.5,\n max_over_time(process_resident_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}\n)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "median", - "range": true, - "refId": "C" - } - ], - "title": "Memory usage % ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "99th percentile of the number of time series read per query.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1360 - }, - "id": 178, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "series", - "range": true, - "refId": "A" - } - ], - "title": "Series read per query ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "99th percentile of number of [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) read per queried time series.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1360 - }, - "id": 180, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "datapoints", - "range": true, - "refId": "A" - } - ], - "title": "Datapoints read per series ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "99th percentile of number of [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) read per query.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1368 - }, - "id": 179, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "datapoints", - "range": true, - "refId": "A" - } - ], - "title": "Datapoints read per query ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "99th percentile of number of [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) scanner per query.\n\nThis number can exceed number of DatapointsReadPerQuery if `step` query arg passed to [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) is smaller than the lookbehind window set in square brackets of [rollup function](https://docs.victoriametrics.com/metricsql/#rollup-functions). For example, if `increase(some_metric[1h])` is executed with the `step=5m`, then the same [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on a hour time range are scanned `1h/5m=12` times. See [this article](https://valyala.medium.com/how-to-optimize-promql-and-metricsql-queries-85a1b75bf986) for details.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1368 - }, - "id": 181, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "datapoints", - "range": true, - "refId": "A" - } - ], - "title": "Datapoints scanned per query ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows network usage between vmselects and clients, such as vmalert, Grafana, vmui, etc.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1376 - }, - "id": 93, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "read from client", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "write to client", - "range": true, - "refId": "B" - } - ], - "title": "Network usage: clients ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows network usage between vmselects and vmstorages.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1376 - }, - "id": 207, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_tcpdialer_read_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "read from vmstorage", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "write to vmstorage", - "range": true, - "refId": "B" - } - ], - "title": "Network usage: vmstorage ($instance)", - "type": "timeseries" - } - ], - "title": "vmselect ($instance)", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 42 - }, - "id": 40, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1345 - }, - "id": 97, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_http_requests_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Requests rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the max number of ongoing insertions.\n* `max` - equal to number of CPU * 2 by default. May be configured with `maxConcurrentInserts` flag;\n* `current` - current number of goroutines busy with processing requests.\n\n`-maxConcurrentInserts` limits the number of insert requests which may be actively processed at any given point in time. All the other insert requests are queued for up to `-insert.maxQueueDuration` in the hope they will get a chance to be processed. This queue is used mostly for absorbing spikes for incoming insert request rate.\n\nWhen `current` hits `max` constantly, it means vminsert node is overloaded and requires more CPU or higher limits.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "max" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C4162A", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1345 - }, - "id": 99, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(max_over_time(vm_concurrent_insert_current{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval]))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "current", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "exemplar": true, - "expr": "min(vm_concurrent_insert_capacity{job=~\"$job_insert\", instance=~\"$instance\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "B" - } - ], - "title": "Concurrent inserts ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=$job_insert&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1353 - }, - "id": 185, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_insert\", instance=~\"$instance\"}\n)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "max", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "min(\n rate(process_cpu_seconds_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_insert\", instance=~\"$instance\"}\n)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "min", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "quantile(0.5,\n rate(process_cpu_seconds_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_insert\", instance=~\"$instance\"}\n)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "median", - "range": true, - "refId": "C" - } - ], - "title": "CPU usage % ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=$job_insert&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1353 - }, - "id": 187, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}\n)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "max", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "min(\n max_over_time(process_resident_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}\n)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "min", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "quantile(0.5,\n max_over_time(process_resident_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}\n)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "median", - "range": true, - "refId": "C" - } - ], - "title": "Memory usage % ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the saturation level of connection between vminsert and vmstorage components. \n\nIf the threshold of 0.9sec is reached, then the connection is saturated by more than 90% and vminsert won't be able to keep up. This usually means that either vminsert or vmstorage nodes are struggling with the load. Verify CPU/mem saturation of both components and network saturation between them.\nIf vminsert resources are saturated - consider adding more resources or scale vminserts horizontally.\n\nIf vminsert resources and network are fine, check vmstorage metrics for anomalies.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent" - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1361 - }, - "id": 139, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(rate(vm_rpc_send_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Storage connection saturation ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows when vmstorage node is unreachable for vminsert.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1361 - }, - "id": 114, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "vm_rpc_vmstorage_is_reachable{job=~\"$job\", instance=~\"$instance\"} != 1", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}} => {{addr}}", - "range": true, - "refId": "A" - } - ], - "title": "Storage reachability ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows network usage between vminserts and clients, such as vmagent, Prometheus, or any other client pushing metrics to vminsert.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1369 - }, - "id": 208, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "read from client", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "write to client", - "range": true, - "refId": "B" - } - ], - "title": "Network usage: clients ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows network usage between vminserts and vmstorages.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1369 - }, - "id": 209, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_tcpdialer_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "read from vmstorage", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "write to vmstorage", - "range": true, - "refId": "B" - } - ], - "title": "Network usage: vmstorage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1377 - }, - "id": 88, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(histogram_quantile(0.99, sum(increase(vm_rows_per_insert_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "max", - "range": true, - "refId": "A" - } - ], - "title": "Rows per insert ($instance)", - "type": "timeseries" - } - ], - "title": "vminsert ($instance)", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 194, - "panels": [ - { - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 1394 - }, - "id": 198, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.", - "mode": "markdown" - }, - "pluginVersion": "9.2.7", - "title": "", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1396 - }, - "id": 189, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{instance}} ({{job}})", - "range": true, - "refId": "A" - } - ], - "title": "RSS memory usage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "RSS share for memory allocated by the process itself. This share cannot be freed by the OS, so it must be taken into account by OOM killer.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1396 - }, - "id": 190, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{instance}} ({{job}})", - "range": true, - "refId": "A" - } - ], - "title": "RSS anonymous memory usage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 1404 - }, - "id": 192, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}} ({{job}})", - "range": true, - "refId": "A" - } - ], - "title": "CPU usage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the approx time needed to reach 100% of disk capacity based on the following params:\n* free disk space (after -storage.minFreeDiskSpaceBytes);\n* row ingestion rate;\n* compression.\n\nNote: this panel doesn't account for deduplication process.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 1404 - }, - "id": 196, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "min"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "(vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) \n/ \nignoring(path) (\n rate(vm_rows_added_to_storage_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d])\n * scalar(\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n / \n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n )\n)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Storage full ETA ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1411 - }, - "id": 200, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "Disk space usage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1411 - }, - "id": 201, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(job, instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{job}}:{{instance}} (indexdb)", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"}) by(job, instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{job}}:{{instance}} (datapoints)", - "range": true, - "refId": "B" - } - ], - "title": "Disk space usage by type ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1419 - }, - "id": 203, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, instance, level, location) > 0", - "format": "time_series", - "hide": false, - "interval": "5m", - "intervalFactor": 1, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Logging rate", - "type": "timeseries" - } - ], - "title": "Drilldown", - "type": "row" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "VictoriaMetrics - cluster", - "value": "PAF93674D0B4E9963" - }, - "includeAll": false, - "name": "ds", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "definition": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)", - "includeAll": true, - "multi": true, - "name": "job", - "options": [], - "query": { - "query": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)", - "refId": "VictoriaMetrics-job-Variable-Query" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vminsert.*\"}, job)", - "hide": 2, - "includeAll": true, - "multi": true, - "name": "job_insert", - "options": [], - "query": { - "query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vminsert.*\"}, job)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmselect.*\"}, job)", - "hide": 2, - "includeAll": true, - "multi": true, - "name": "job_select", - "options": [], - "query": { - "query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmselect.*\"}, job)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmstorage.*\"}, job)", - "hide": 2, - "includeAll": true, - "multi": true, - "name": "job_storage", - "options": [], - "query": { - "query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmstorage.*\"}, job)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "allValue": ".*", - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "definition": "label_values(vm_app_version{job=~\"$job\"}, instance)", - "includeAll": true, - "multi": true, - "name": "instance", - "options": [], - "query": { - "query": "label_values(vm_app_version{job=~\"$job\"}, instance)", - "refId": "VictoriaMetrics-instance-Variable-Query" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "baseFilters": [], - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "filters": [], - "name": "adhoc", - "type": "adhoc" - } - ] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "VictoriaMetrics - cluster", - "uid": "oS7Bi_0Wz", - "version": 1, - "weekStart": "", - "gnetId": 11176 - }, - "overwrite": true -} diff --git a/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_vmagent.json b/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_vmagent.json deleted file mode 100644 index 39f8d22..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/drycc_victoriametrics_vmagent.json +++ /dev/null @@ -1,7252 +0,0 @@ -{ - "dashboard": { - "__inputs": [], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "10.4.2" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - }, - { - "type": "panel", - "id": "text", - "name": "Text", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "enable": true, - "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(short_version))", - "hide": true, - "iconColor": "dark-blue", - "name": "version", - "textFormat": "{{short_version}}", - "titleFormat": "Version change" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "enable": true, - "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job, instance)", - "iconColor": "dark-yellow", - "name": "restarts", - "textFormat": "{{job}}:{{instance}} restarted" - } - ] - }, - "description": "Overview for VictoriaMetrics vmagent v1.102.0 or higher", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "links": [ - { - "icon": "doc", - "tags": [], - "targetBlank": true, - "title": "vmagent wiki", - "tooltip": "", - "type": "link", - "url": "https://docs.victoriametrics.com/vmagent/" - }, - { - "icon": "external link", - "tags": [], - "targetBlank": true, - "title": "Found a bug?", - "type": "link", - "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues" - }, - { - "icon": "external link", - "tags": [], - "targetBlank": true, - "title": "New releases", - "type": "link", - "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases" - } - ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 105, - "panels": [], - "title": "Stats", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the rate of [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) scraped from configured targets.", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 103, - "links": [ - { - "title": "Troubleshooting", - "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" - } - ], - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_promscrape_scraped_samples_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Samples scraped/s", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the rate of ingested [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 102, - "links": [ - { - "title": "Troubleshooting", - "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" - } - ], - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vmagent_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Samples ingested/s", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of targets scraped per second.", - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 1 - }, - "id": 134, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_promscrape_scrapes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) ", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Targets scraped/s", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows total number of all configured scrape targets in state `up` or `down`.\n\nSee `http://vmagent-host:8429/targets` to get list of all targets. \n", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "down" - }, - "properties": [ - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 1 - }, - "id": 72, - "links": [ - { - "title": "Troubleshooting", - "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" - } - ], - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"up\"})", - "interval": "", - "legendFormat": "up", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"down\"})", - "hide": false, - "interval": "", - "legendFormat": "down", - "range": true, - "refId": "B" - } - ], - "title": "Scrape targets", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows number of generated error messages in logs over last 30m. Non-zero value may be a sign of connectivity or missconfiguration errors.", - "fieldConfig": { - "defaults": { - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 1 - }, - "id": 16, - "links": [ - { - "targetBlank": true, - "title": "Troubleshooting", - "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" - } - ], - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "sum(increase(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[30m]))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Log errors (30m)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Persistent queue size shows size of pending samples in bytes which hasn't been flushed to remote storage yet. \nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 10485760 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 56, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "sum(vm_persistentqueue_bytes_pending{job=~\"$job\", instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Persistent queue size", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "minWidth": 50 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "displayName", - "value": "Count" - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 4 - }, - "id": 101, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)", - "format": "table", - "instant": true, - "range": false, - "refId": "A" - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepAfter", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 16, - "x": 8, - "y": 4 - }, - "id": 13, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(up{job=~\"$job\", instance=~\"$instance\"}) by (job)", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "__auto", - "refId": "A" - } - ], - "title": "Uptime", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 24, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "refId": "A" - } - ], - "title": "Overview", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows in/out [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) rate including push and pull models. \n\nThe out-rate could be different to in-rate because of replication or additional timeseries added by vmagent for every scraped target.\n\nClick on the line and choose Drilldown to show CPU usage per instance\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - { - "title": "Drilldown", - "url": "/d/G7Z9GzMGz?viewPanel=123&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/out .*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate({__name__=~\"vmagent_rows_inserted_total|vm_promscrape_scraped_samples_sum\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", - "hide": false, - "interval": "", - "legendFormat": "in {{job}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vmagent_remotewrite_block_size_rows_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", - "interval": "", - "legendFormat": "out {{job}}", - "range": true, - "refId": "B" - } - ], - "title": "Samples rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the persistent queue size of pending samples in bytes >2MB which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\nClick on the line and choose Drilldown to show the persistent queue size per instance.\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/G7Z9GzMGz?viewPanel=125&var-url=${__field.labels.url}&var-ds=$ds&var-instance=$instance&var-job=$job&${__url_time_range}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 17, - "links": [ - { - "title": "Troubleshooting", - "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" - } - ], - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(vmagent_remotewrite_pending_data_bytes{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}) by (job, url) > 2e6", - "interval": "", - "legendFormat": "{{job}} => {{url}}", - "range": true, - "refId": "A" - } - ], - "title": "Persistent queue size ($instance) to ($url)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 107, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, level) > 0", - "format": "time_series", - "hide": false, - "interval": "5m", - "intervalFactor": 1, - "legendFormat": "{{job}} - {{level}}", - "range": true, - "refId": "A" - } - ], - "title": "Logging rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows rate of dropped [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) from persistent queue. vmagent drops samples from queue if in-memory and on-disk queues are full and it is unable to flush them to remote storage.\nThe max size of on-disk queue is configured by `-remoteWrite.maxDiskUsagePerURL` flag.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 49, - "links": [ - { - "targetBlank": true, - "title": "Troubleshooting", - "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" - } - ], - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_persistentqueue_bytes_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0", - "interval": "", - "legendFormat": "{{ path }}", - "range": true, - "refId": "A" - } - ], - "title": "Persistent queue dropped rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the rate of requests served by vmagent HTTP server.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 26 - }, - "id": 15, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vmagent_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, path) > 0", - "interval": "", - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "Requests rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Errors rate shows rate for multiple metrics that track possible errors in vmagent, such as network or parsing errors.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 26 - }, - "id": 69, - "links": [ - { - "targetBlank": true, - "title": "Troubleshooting", - "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" - } - ], - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vmagent_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, protocol) > 0", - "interval": "", - "legendFormat": "requests:{{protocol}} ({{job}})", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_protoparser_read_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", - "interval": "", - "legendFormat": "parse: {{type}} ({{job}})", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_ingestserver_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", - "interval": "", - "legendFormat": "ingest: {{type}} ({{job}})", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_protoparser_unmarshal_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", - "interval": "", - "legendFormat": "unmarshal: {{type}} ({{job}})", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_promscrape_dial_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", - "interval": "", - "legendFormat": "scrape dial ({{job}})", - "range": true, - "refId": "E" - } - ], - "title": "Errors rate ($instance)", - "type": "timeseries" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 34 - }, - "id": 45, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Click on the line and choose Drilldown to show CPU usage per instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/G7Z9GzMGz?viewPanel=119&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 3 - }, - "id": 109, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "CPU ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Percentage of used RSS memory (resident).\nThe RSS memory shows the amount of memory recently accessed by the application. It includes anonymous memory and data from recently accessed files (aka page cache).\nThe application's performance will significantly degrade when memory usage is close to 100%.\n\nClick on the line and choose Drilldown to show memory usage per instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/G7Z9GzMGz?viewPanel=117&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 3 - }, - "id": 111, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "RSS memory % usage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of bytes read/write from the storage layer when vmagent has to buffer data on disk or read already buffered data.\n\nClick on the line and choose Drilldown to show CPU usage per instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Drilldown", - "url": "/d/G7Z9GzMGz?viewPanel=121&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" - } - ], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "read" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 81, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "read {{job}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "write {{job}}", - "range": true, - "refId": "B" - } - ], - "title": "Disk writes/reads ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Network usage shows the bytes rate for data accepted by vmagent and pushed via remotewrite protocol.\nDiscrepancies are possible because of different protocols used for ingesting, scraping and writing data.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/out.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) * 8 \n+ sum(rate(vm_promscrape_conn_bytes_read_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) * 8", - "interval": "", - "legendFormat": "in {{job}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vmagent_remotewrite_conn_bytes_written_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) * 8", - "interval": "", - "legendFormat": "out {{job}}", - "range": true, - "refId": "B" - } - ], - "title": "Network usage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Panel shows the percentage of open file descriptors in the OS per instance.\nReaching the limit of open files (100%) can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 5, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 19 - }, - "id": 83, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(\n max_over_time(process_open_fds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_max_fds{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Open FDs usage % ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 19 - }, - "id": 39, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(max_over_time(go_goroutines{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Goroutines ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/troubleshooting/ ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 27 - }, - "id": 135, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(\n rate(go_gc_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n ) by(job)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "CPU spent on GC ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.1 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 27 - }, - "id": 149, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Go scheduling latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 35 - }, - "id": 41, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(max_over_time(process_num_threads{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Threads ($instance)", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "refId": "A" - } - ], - "title": "Resource usage", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "id": 94, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows top 10 jobs by the number of new series registered by vmagent over the 5min range. These jobs generate the most of the churn rate.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 4 - }, - "id": 92, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "topk_max(10, sum(sum_over_time(scrape_series_added[5m])) by (job)) > 0", - "interval": "", - "legendFormat": "{{ job }}", - "range": true, - "refId": "A" - } - ], - "title": "Top 10 jobs by unique samples", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows top 10 instances by the number of new series registered by vmagent over the 5min range. These instances generate the most of the churn rate.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 4 - }, - "id": 95, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "topk(10, sum(sum_over_time(scrape_series_added[5m])) by (instance)) > 0", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Top 10 instances by unique samples", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows write saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 2, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 - }, - "id": 98, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(rate(vm_persistentqueue_write_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Persistent queue write saturation ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows read saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with reading data from the disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 2, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 - }, - "id": 99, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(rate(vm_persistentqueue_read_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Persistent queue read saturation ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the rate of dropped data blocks in cases when remote storage replies with `400 Bad Request` and `409 Conflict` HTTP responses.\n\nSee https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 20 - }, - "id": 79, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vmagent_remotewrite_packets_dropped_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) by(job, url) > 0", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Data blocks dropped ($instance) to ($url)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the rate of dropped [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) due to relabeling. \nMetric tracks drops for `-remoteWrite.relabelConfig` configuration only.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 20 - }, - "id": 18, - "links": [ - { - "targetBlank": true, - "title": "Relabeling", - "url": "https://docs.victoriametrics.com/vmagent/#relabeling" - } - ], - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vmagent_remotewrite_relabel_metrics_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, url) > 0", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "B" - } - ], - "title": "Rows dropped by relabeling ($instance) to ($url)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the rate of parsed datapoints from write or scrape requests.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 127, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vm_protoparser_rows_read_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", - "interval": "", - "legendFormat": "{{ type }} ({{job}})", - "range": true, - "refId": "A" - } - ], - "title": "Datapoints rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Tracks the rate of dropped invalid rows because of errors while unmarshaling write requests. The exact errors messages will be printed in logs.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 28 - }, - "id": 50, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vm_rows_invalid_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", - "interval": "", - "legendFormat": "{{type}} ({{job}})", - "range": true, - "refId": "A" - } - ], - "title": "Invalid datapoints rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 36 - }, - "id": 129, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "job" - } - ] - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(flag{is_set=\"true\", job=~\"$job\", instance=~\"$instance\"}) by(job, instance, name, value)", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Non-default flags", - "transformations": [ - { - "id": "groupBy", - "options": { - "fields": { - "instance": { - "aggregations": [] - }, - "job": { - "aggregations": [], - "operation": "groupby" - }, - "name": { - "aggregations": [], - "operation": "groupby" - }, - "value": { - "aggregations": [], - "operation": "groupby" - } - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepAfter", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 36 - }, - "id": 150, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)", - "format": "time_series", - "instant": false, - "legendFormat": "{{job}}", - "refId": "A" - } - ], - "title": "Restarts ($job)", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Troubleshooting", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 28, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 48, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"up\"}) by(job, type) > 0", - "format": "time_series", - "interval": "", - "legendFormat": "{{type}} ({{job}})", - "range": true, - "refId": "A" - } - ], - "title": "Scrape targets UP(By Type)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 53 - }, - "id": 76, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"down\"}) by(job, type) > 0", - "format": "time_series", - "interval": "", - "legendFormat": "{{type}} ({{job}})", - "range": true, - "refId": "A" - } - ], - "title": "Scrape targets DOWN(By Type)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 60 - }, - "id": 132, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(vm_promscrape_scrape_pool_targets{job=~\"$job\", instance=~\"$instance\", status=\"up\"}) by(job, scrape_job) > 0", - "format": "time_series", - "interval": "", - "legendFormat": "{{job}}: {{scrape_job}}", - "range": true, - "refId": "A" - } - ], - "title": "Scrape targets UP(By Job)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 60 - }, - "id": 133, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(vm_promscrape_scrape_pool_targets{job=~\"$job\", instance=~\"$instance\", status=\"down\"}) by(job, scrape_job) > 0", - "format": "time_series", - "interval": "", - "legendFormat": "{{job}}: {{scrape_job}}", - "range": true, - "refId": "A" - } - ], - "title": "Scrape targets DOWN(By Job)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of scrapes per second.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 20, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vm_promscrape_scrapes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Scrape rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of datapoints scraped per second.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 126, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vm_promscrape_scraped_samples_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Scraped datapoints rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 75 - }, - "id": 46, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_response_size_bytes_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)", - "format": "time_series", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Scrape response size 0.99 quantile ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 75 - }, - "id": 148, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)", - "format": "time_series", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Scrape duration 0.99 quantile ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 83 - }, - "id": 31, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_promscrape_scrapes_failed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", - "interval": "", - "legendFormat": "scrapes failed ({{job}})", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_promscrape_scrapes_timed_out_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", - "interval": "", - "legendFormat": "timeouts ({{job}})", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_promscrape_scrapes_gunzip_failed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", - "interval": "", - "legendFormat": "gunzip fails ({{job}})", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_promscrape_dial_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", - "interval": "", - "legendFormat": "dial fails ({{job}})", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_promscrape_max_scrape_size_exceeded_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", - "hide": false, - "interval": "", - "legendFormat": "max scrape size exceeded ({{job}})", - "range": true, - "refId": "E" - } - ], - "title": "Scrape fails ($instance)", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "refId": "A" - } - ], - "title": "Scraping", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 37 - }, - "id": 71, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the rate of write requests served by ingestserver (UDP, TCP connections) and HTTP server.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 30 - }, - "id": 73, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vm_ingestserver_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type, net) > 0", - "interval": "", - "legendFormat": "{{net}}: {{ type }} ({{job}})", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vmagent_http_requests_total{job=~\"$job\", instance=~\"$instance\", protocol!=\"\"}[$__rate_interval])) by(job, protocol) > 0", - "interval": "", - "legendFormat": "{{ protocol }}: http ({{job}})", - "range": true, - "refId": "B" - } - ], - "title": "Requests rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the rate of rows ingested in vmagent via push protocols.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 22 - }, - "id": 131, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vmagent_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", - "interval": "", - "legendFormat": "{{ type }} ({{job}})", - "range": true, - "refId": "A" - } - ], - "title": "Rows rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows how many concurrent inserts are taking place.\n\nIf the number of concurrent inserts hitting the `limit` or is close to the `limit` constantly - it might be a sign of a resource shortage.\n\n If vmagent's CPU usage and remote write connection saturation are at normal level, it might be that `-maxConcurrentInserts` cmd-line flag need to be increased.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 30 - }, - "id": 130, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max_over_time(vm_concurrent_insert_current{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{instance}} ({{job}})", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "min(vm_concurrent_insert_capacity{job=~\"$job\", instance=~\"$instance\"}) by(job)", - "interval": "", - "legendFormat": "limit ({{job}})", - "range": true, - "refId": "B" - } - ], - "title": "Concurrent inserts ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the rate of write errors in ingestserver (UDP, TCP connections) and HTTP server.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 30 - }, - "id": 77, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "exemplar": true, - "expr": "sum(rate(vm_ingestserver_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(type, net) > 0", - "interval": "", - "legendFormat": "{{ type }} ({{net}})", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "exemplar": true, - "expr": "sum(rate(vmagent_http_request_errors_total{job=~\"$job\", instance=~\"$instance\", protocol!=\"\"}[$__rate_interval])) by(protocol) > 0", - "interval": "", - "legendFormat": "{{ protocol }} (http)", - "refId": "B" - } - ], - "title": "Error rate ($instance)", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "refId": "A" - } - ], - "title": "Ingestion", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 38 - }, - "id": 136, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of matched samples by the aggregation rule. \n\nThe more samples is matched, the more work this aggregation rule does. The matching rule is specified via `match` param.\n\nSee more details in [stream aggregation config](https://docs.victoriametrics.com/stream-aggregation/#stream-aggregation-config). ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 146, - "options": { - "legend": { - "calcs": ["min", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_streamaggr_matched_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Matched samples ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "The rate of ignored samples during aggregation. \nStream aggregation will drop samples with NaN values, or samples with too old timestamps. See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 23 - }, - "id": 143, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_streamaggr_ignored_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0) without (instance, pod)", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Ignored samples ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of produced samples by the aggregation rule. \n\nNumber of produced samples depend on params like `by`, `without`, `interval`, etc.\n\nSee more details in [stream aggregation config](https://docs.victoriametrics.com/stream-aggregation/#stream-aggregation-config). ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 31 - }, - "id": 147, - "options": { - "legend": { - "calcs": ["min", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_streamaggr_output_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Produced samples ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows events when deduplication or aggregation couldn't be finished in the configured interval. Such events may result into bad accuracy of produced data.\n\nPossible solutions:\n* increase interval; \n* use match filter matching smaller number of series;\n* reduce samples ingestion rate to stream aggregation", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": -5, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 31 - }, - "id": 139, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0", - "instant": false, - "legendFormat": "aggregation: {{url}} ({{job}}): {{path}}:{{position}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0", - "hide": false, - "instant": false, - "legendFormat": "deduplication: {{url}} ({{job}}): {{path}}:{{position}}", - "range": true, - "refId": "B" - } - ], - "title": "Flush timeouts ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the max lag between samples timestamps within one batch passed to the aggregation input.\n\nLower is better.\n\nToo high lag or lag exceeding the interval might be a sign that data was delayed before aggregation or resource insufficiency on aggregator. Samples with high lag may affect accuracy of aggregation.\n\nSee https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 142, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod))", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Samples lag 0.99 quantile ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "The 99th percentile of avg flush duration for the aggregated data. \n\nSmaller is better.\n\nAggregation can produce incorrect results ff flush duration exceeds configured deduplication interval. See \"Flush Timeouts\" panel.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 137, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, rate(vm_streamaggr_dedup_flush_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Dedup flush duration 0.99 quantile ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the size of Label Compressor in number of entries.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/bytes.*/" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 141, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(vm_streamaggr_labels_compressor_items_count{job=~\"$job\",instance=~\"$instance\"}) by(job, instance)", - "hide": false, - "instant": false, - "legendFormat": "items: {{instance}} ({{job}})", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max(vm_streamaggr_labels_compressor_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)", - "hide": false, - "instant": false, - "legendFormat": "bytes: {{instance}} ({{job}})", - "range": true, - "refId": "B" - } - ], - "title": "Labels compressor ($instance)", - "type": "timeseries" - } - ], - "title": "Streaming aggregation", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 39 - }, - "id": 58, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the rate of requests to configured remote write endpoints by url and status code.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 60, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vmagent_remotewrite_requests_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) by(job, url, status_code) > 0", - "interval": "", - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "Requests rate ($instance) to ($url)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the global rate for number of written bytes via remote write connections.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 66, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vmagent_remotewrite_conn_bytes_written_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Bytes write rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows requests retry rate by url. Number of retries is unlimited but protected with delays up to 1m between attempts.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 61, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(vmagent_remotewrite_retries_count_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) by(url) > 0", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Retry rate ($instance) to ($url)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows current number of established connections to remote write endpoints.\n\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 65, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(max_over_time(vmagent_remotewrite_conns{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Connections ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the current limit usage of unique series over an hourly period. Vmagent will start to drop series once the limit is reached.\n\nPlease note, panel will be blank if `remoteWrite.maxHourlySeries` is not set.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent" - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 88, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n vmagent_hourly_series_limit_current_series{job=~\"$job\", instance=~\"$instance\"} \n / \n vmagent_hourly_series_limit_max_series{job=~\"$job\", instance=~\"$instance\"}\n ) by(job) * 100", - "interval": "", - "legendFormat": "{{job}}", - "range": true, - "refId": "A" - } - ], - "title": "Hourly series limit", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows saturation of every connection to remote storage. If the threshold of 90% is reached, then the connection is saturated (busy or slow) by more than 90%, so vmagent won't be able to keep up and can start buffering data. \n\nThis usually means that `-remoteWrite.queues` command-line flag must be increased in order to increase the number of connections per each remote storage.\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent" - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 84, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n rate(vmagent_remotewrite_send_duration_seconds_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])\n /\n vmagent_remotewrite_queues{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}\n) by(job, url)", - "interval": "", - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "Remote write connection saturation ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the current limit usage of unique series over a daily period. Vmagent will start to drop series once the limit is reached.\n\nPlease note, panel will be blank if `remoteWrite.maxDailySeries` is not set.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent" - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 90, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(\n vmagent_daily_series_limit_current_series{job=~\"$job\",instance=~\"$instance\"} \n / \n vmagent_daily_series_limit_max_series{job=~\"$job\",instance=~\"$instance\"}\n) by(job)", - "interval": "", - "legendFormat": "{{job}}", - "range": true, - "refId": "A" - } - ], - "title": "Daily series limit", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "refId": "A" - } - ], - "title": "Remote write", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 113, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 115, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.", - "mode": "markdown" - }, - "pluginVersion": "10.4.2", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 35 - }, - "id": 119, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}} ({{job}})", - "range": true, - "refId": "A" - } - ], - "title": "CPU usage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 35 - }, - "id": 117, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{instance}} ({{job}})", - "range": true, - "refId": "A" - } - ], - "title": "RSS memory usage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the persistent queue size of pending samples in bytes which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 43 - }, - "id": 125, - "links": [ - { - "title": "Troubleshooting", - "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" - } - ], - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(vmagent_remotewrite_pending_data_bytes{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}) by (instance, url)", - "interval": "", - "legendFormat": "{{instance}} => {{url}}", - "range": true, - "refId": "A" - } - ], - "title": "Persistent queue size ($instance) to ($url)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows in/out samples rate including push and pull models. \n\nThe out-rate could be different to in-rate because of replication or additional timeseries added by vmagent for every scraped target.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/out .*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 43 - }, - "id": 123, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_promscrape_scraped_samples_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)\n+ sum(rate(vmagent_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", - "hide": false, - "interval": "", - "legendFormat": "in {{instance}} {{job}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vmagent_remotewrite_block_size_rows_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", - "interval": "", - "legendFormat": "out {{instance}} {{job}}", - "range": true, - "refId": "B" - } - ], - "title": "Samples rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of bytes read/write from the storage layer when vmagent has to buffer data on disk or read already buffered data.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "read" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 51 - }, - "id": 121, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "read {{instance}} {{job}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job,instance)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "write {{instance}} {{job}}", - "range": true, - "refId": "B" - } - ], - "title": "Disk writes/reads ($instance)", - "type": "timeseries" - } - ], - "title": "Drilldown", - "type": "row" - } - ], - "refresh": "", - "schemaVersion": 39, - "tags": ["vmagent", "victoriametrics"], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "VictoriaMetrics - cluster", - "value": "PAF93674D0B4E9963" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "ds", - "options": [], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": "", - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "definition": "label_values(vm_app_version{version=~\"^vmagent.*\"}, job)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "job", - "options": [], - "query": { - "query": "label_values(vm_app_version{version=~\"^vmagent.*\"}, job)", - "refId": "VictoriaMetrics-job-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "definition": "label_values(vm_app_version{job=~\"$job\"}, instance)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "instance", - "options": [], - "query": { - "query": "label_values(vm_app_version{job=~\"$job\"}, instance)", - "refId": "VictoriaMetrics-instance-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "definition": "label_values(vmagent_remotewrite_requests_total{job=~\"$job\", instance=~\"$instance\"}, url)", - "description": "The remote write URLs", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "url", - "options": [], - "query": { - "query": "label_values(vmagent_remotewrite_requests_total{job=~\"$job\", instance=~\"$instance\"}, url)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "filters": [], - "hide": 0, - "name": "adhoc", - "skipUrlSync": false, - "type": "adhoc" - } - ] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "VictoriaMetrics - vmagent", - "uid": "G7Z9GzMGz", - "version": 1, - "weekStart": "", - "gnetId": 12683 - }, - "overwrite": true -} diff --git a/rootfs/usr/share/grafana/api/dashboards/kubernetes.json b/rootfs/usr/share/grafana/api/dashboards/kubernetes.json deleted file mode 100644 index 8815e47..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/kubernetes.json +++ /dev/null @@ -1,5929 +0,0 @@ -{ - "dashboard": { - "annotations": { - "list": [ - { - "$$hashKey": "object:247", - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "【English version】2025.01.25 Updates，kubernetesComprehensive display of resources！Includes K8S Overall Resource Overview、Microservices Resource Details、Pod Resource Details andK8SNetwork Bandwidth，Optimization metrics。https://grafana.com/orgs/starsliao/dashboards", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [ - { - "icon": "bolt", - "tags": [], - "targetBlank": true, - "title": "Update", - "tooltip": "See more dashboards", - "type": "link", - "url": "https://grafana.com/orgs/starsliao/dashboards" - }, - { - "$$hashKey": "object:831", - "icon": "question", - "tags": ["node_exporter"], - "targetBlank": true, - "title": "GitHub", - "tooltip": "View more dashboards", - "type": "link", - "url": "https://github.com/starsliao" - } - ], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 54, - "panels": [], - "title": "Node Resource Overview：Selected Nodes:【$Node】", - "type": "row" - }, - { - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "orange", - "value": 0.8 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 44, - "options": { - "displayMode": "basic", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 10, - "minVizWidth": 0, - "namePlacement": "auto", - "orientation": "vertical", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showUnfilled": false, - "sizing": "auto", - "text": {}, - "valueMode": "color" - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "sum(container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Memory Utilization", - "refId": "C" - }, - { - "editorMode": "code", - "exemplar": true, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Memory Request Rate", - "refId": "A" - }, - { - "editorMode": "code", - "exemplar": true, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Memory Limit Rate", - "refId": "B", - "step": 10 - } - ], - "title": "Node Memory Ratio", - "type": "bargauge" - }, - { - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "orange", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 45, - "options": { - "displayMode": "basic", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 10, - "minVizWidth": 0, - "namePlacement": "auto", - "orientation": "vertical", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showUnfilled": false, - "sizing": "auto", - "text": {}, - "valueMode": "color" - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m])) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "CPU Utilization Rate", - "refId": "C" - }, - { - "editorMode": "code", - "exemplar": true, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", - "format": "time_series", - "instant": true, - "interval": "", - "legendFormat": "CPU Request Rate", - "refId": "A" - }, - { - "editorMode": "code", - "exemplar": true, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", - "format": "time_series", - "instant": true, - "interval": "", - "legendFormat": "CPU Limit Rate", - "refId": "B" - } - ], - "title": "Node CPU Ratio", - "type": "bargauge" - }, - { - "description": "Number of cluster nodes，Nodes POD Number of，NodesPODUpper Limit", - "fieldConfig": { - "defaults": { - "mappings": [], - "max": 100, - "min": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "orange", - "value": 1000 - }, - { - "color": "red", - "value": 2000 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 8, - "y": 1 - }, - "id": 74, - "options": { - "displayMode": "basic", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 10, - "minVizWidth": 0, - "namePlacement": "auto", - "orientation": "vertical", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showUnfilled": false, - "sizing": "auto", - "text": {}, - "valueMode": "color" - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "count(kube_node_info{origin_prometheus=~\"$origin_prometheus\"})", - "instant": true, - "interval": "", - "legendFormat": "Number of nodes", - "refId": "A" - }, - { - "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",created_by_kind!~\"|Job\",node=~\"^$Node$\"})", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Pod Number of nodes", - "refId": "B" - }, - { - "exemplar": true, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"pods\", unit=\"integer\",node=~\"^$Node$\"})", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Upper limit Pod", - "refId": "C" - } - ], - "title": "Nodes with Pod", - "type": "bargauge" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "color-text" - }, - "filterable": false, - "inspect": false - }, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Space" - }, - "properties": [ - { - "id": "custom.width", - "value": 59 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Pod" - }, - "properties": [ - { - "id": "custom.width", - "value": 21 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "SVC" - }, - "properties": [ - { - "id": "custom.width", - "value": 7 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Microservices" - }, - "properties": [ - { - "id": "custom.width", - "value": 4 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Configuration" - }, - "properties": [ - { - "id": "custom.width", - "value": 16 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Passwords" - }, - "properties": [ - { - "id": "custom.width", - "value": 33 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 5, - "x": 11, - "y": 1 - }, - "id": 51, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": true - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Microservices" - } - ] - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\"}) by (namespace)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "count(kube_service_info{origin_prometheus=~\"$origin_prometheus\"}) by(namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "C" - }, - { - "editorMode": "code", - "expr": "count by (namespace)({__name__=~\"kube_deployment_metadata_generation|kube_daemonset_metadata_generation|kube_statefulset_metadata_generation\",origin_prometheus=~\"$origin_prometheus\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "__auto", - "refId": "D" - }, - { - "editorMode": "code", - "expr": "count(kube_configmap_info{origin_prometheus=~\"$origin_prometheus\"}) by(namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "configmap", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "count(kube_secret_info{origin_prometheus=~\"$origin_prometheus\"}) by(namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "secret", - "refId": "E" - } - ], - "title": "Namespace Resource Statistics", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "namespace" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true - }, - "includeByName": {}, - "indexByName": { - "Time 1": 2, - "Time 2": 4, - "Time 3": 6, - "Value #A": 3, - "Value #C": 5, - "Value #D": 1, - "namespace": 0 - }, - "renameByName": { - "Time 1": "", - "Time 2": "", - "Value #A": "Pod", - "Value #B": "Configuration", - "Value #C": "SVC", - "Value #D": "Microservices", - "Value #E": "Passwords", - "namespace": "Spaces" - } - } - } - ], - "type": "table" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "series", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 4, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binbps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 1 - }, - "id": 32, - "options": { - "legend": { - "calcs": ["max"], - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "width": 200 - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "sum (irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m]))*8", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Receive", - "metric": "network", - "range": true, - "refId": "A", - "step": 10 - }, - { - "editorMode": "code", - "exemplar": true, - "expr": "sum (irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m]))*8", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Send", - "metric": "network", - "refId": "B", - "step": 10 - } - ], - "title": "$NameSpace：Network Overview（Associable nodes and namespaces）", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "max": 2000000000000, - "min": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "orange", - "value": 100000000000 - }, - { - "color": "red", - "value": 2000000000000 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 5 - }, - "id": 71, - "options": { - "displayMode": "basic", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 10, - "minVizWidth": 0, - "namePlacement": "auto", - "orientation": "vertical", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showUnfilled": false, - "sizing": "auto", - "text": {}, - "valueMode": "color" - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "exemplar": true, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Total Memory", - "refId": "A" - }, - { - "exemplar": true, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Usage", - "refId": "C" - }, - { - "exemplar": true, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Requests", - "refId": "D" - }, - { - "exemplar": true, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Limits", - "refId": "B" - } - ], - "title": "Node Memory Information", - "type": "bargauge" - }, - { - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "max": 500, - "min": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "#EAB839", - "value": 500 - }, - { - "color": "red", - "value": 1000 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 5 - }, - "id": 72, - "options": { - "displayMode": "basic", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 10, - "minVizWidth": 0, - "namePlacement": "auto", - "orientation": "vertical", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showUnfilled": false, - "sizing": "auto", - "text": {}, - "valueMode": "color" - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "exemplar": true, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Total Cores", - "refId": "A" - }, - { - "exemplar": true, - "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",id=\"/\",node=~\"^$Node$\"}[2m]))", - "instant": true, - "interval": "", - "legendFormat": "Usage", - "refId": "C" - }, - { - "exemplar": true, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Requests", - "refId": "D" - }, - { - "exemplar": true, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Limit", - "refId": "B" - } - ], - "title": "Node CPU Number of cores", - "type": "bargauge" - }, - { - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "max": 8000000000000, - "min": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "#EAB839", - "value": 5000000000000 - }, - { - "color": "red", - "value": 10000000000000 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Utilization" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "max" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "thresholds", - "value": { - "mode": "percentage", - "steps": [ - { - "color": "green" - }, - { - "color": "orange", - "value": 80 - }, - { - "color": "red", - "value": 90 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "decimals", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 8, - "y": 5 - }, - "id": 73, - "options": { - "displayMode": "basic", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 10, - "minVizWidth": 0, - "namePlacement": "auto", - "orientation": "vertical", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showUnfilled": false, - "sizing": "auto", - "text": {}, - "valueMode": "color" - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"}) / sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Utilization Rate", - "refId": "C" - }, - { - "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Usage", - "refId": "A" - }, - { - "expr": "sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Total", - "refId": "B" - } - ], - "title": "Node Storage Information", - "type": "bargauge" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Anomaly.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 88, - "maxPerRow": 2, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": ["last"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.1", - "repeat": "origin_prometheus", - "repeatDirection": "v", - "targets": [ - { - "editorMode": "code", - "exemplar": false, - "expr": "count({__name__=~\"kube_deployment_metadata_generation|kube_daemonset_metadata_generation|kube_statefulset_metadata_generation\",origin_prometheus=~\"$origin_prometheus\"})", - "hide": false, - "instant": true, - "legendFormat": "Workload", - "range": false, - "refId": "F" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\"})", - "hide": false, - "instant": true, - "legendFormat": "Total Pod", - "range": false, - "refId": "E" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "count by(key,origin_prometheus)(kube_node_spec_taint{origin_prometheus=~\"$origin_prometheus\",key=~\"node.kubernetes.io.*\"})", - "format": "time_series", - "hide": false, - "instant": true, - "legendFormat": "{{key}}", - "range": false, - "refId": "D" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "count by(origin_prometheus)(kube_node_info{origin_prometheus=~\"$origin_prometheus\"})", - "hide": false, - "instant": true, - "legendFormat": "Total Nodes", - "range": false, - "refId": "C" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "count by(origin_prometheus)(kube_node_info{origin_prometheus=~\"$origin_prometheus\"}) - count by(origin_prometheus)(kube_node_spec_taint{origin_prometheus=~\"$origin_prometheus\",key!~\"node.kubernetes.io.*\"})", - "hide": false, - "instant": true, - "legendFormat": "Normal Node", - "range": false, - "refId": "B" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "count by(key,origin_prometheus)(kube_node_spec_taint{origin_prometheus=~\"$origin_prometheus\",key!~\"node.kubernetes.io.*\"})", - "hide": false, - "instant": true, - "legendFormat": "{{key}}", - "range": false, - "refId": "A" - } - ], - "title": "", - "transformations": [ - { - "id": "renameByRegex", - "options": { - "regex": "(node.kubernetes.io/)(.*)", - "renamePattern": "Abnormal:$2" - } - } - ], - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Total Memory" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - }, - { - "id": "custom.lineWidth", - "value": 2 - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 11 - }, - "id": 79, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", - "instant": false, - "interval": "", - "legendFormat": "Total Memory", - "refId": "A" - }, - { - "exemplar": true, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})", - "instant": false, - "interval": "", - "legendFormat": "Usage", - "refId": "C" - }, - { - "exemplar": true, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "Requests", - "refId": "D" - }, - { - "exemplar": true, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "Limit", - "refId": "B" - } - ], - "title": "Memory Usage【$Node】", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Total Cores" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - }, - { - "id": "custom.lineWidth", - "value": 2 - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 11 - }, - "id": 80, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", - "instant": false, - "interval": "", - "legendFormat": "Total Cores", - "refId": "A" - }, - { - "exemplar": true, - "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",id=\"/\",node=~\"^$Node$\"}[2m]))", - "instant": false, - "interval": "", - "legendFormat": "Usage", - "refId": "C" - }, - { - "exemplar": true, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "Requests", - "refId": "D" - }, - { - "exemplar": true, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", - "hide": true, - "instant": true, - "interval": "", - "legendFormat": "Limit", - "refId": "B" - } - ], - "title": "CPU Used Cores【$Node】", - "type": "timeseries" - }, - { - "description": "Number of Cluster Nodes，Nodes POD Number of，Nodes POD Upper Limit", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "series", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Upper Limit Pod" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - }, - { - "id": "custom.lineWidth", - "value": 2 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Number of nodes" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.drawStyle", - "value": "points" - }, - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - }, - { - "id": "custom.pointSize", - "value": 3 - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 11 - }, - "id": 81, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "exemplar": true, - "expr": "count(kube_node_info{origin_prometheus=~\"$origin_prometheus\"})", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Number of nodes", - "refId": "A" - }, - { - "editorMode": "code", - "exemplar": true, - "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",created_by_kind!~\"|Job\",node=~\"^$Node$\"})", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Pod Number", - "refId": "B" - }, - { - "exemplar": true, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"pods\", unit=\"integer\",node=~\"^$Node$\"})", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Upper Limit Pod", - "refId": "C" - } - ], - "title": "Pod Number and nodes【$Node】", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Total number of cores.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C4162A", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 16 - }, - "id": 75, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "exemplar": true, - "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m]))by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)*100", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{node}}", - "refId": "I" - } - ], - "title": "$Node：Nodes CPU Breakdown", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 16 - }, - "id": 76, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "exemplar": true, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)*100", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{node}}", - "refId": "I" - } - ], - "title": "$Node：Node Memory Breakdown", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "series", - "axisLabel": "←Inflow/Outflow→", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binbps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Inflow.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 16 - }, - "id": 78, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "exemplar": true, - "expr": "sum (irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\"}[2m]))by (node) *8", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Inflow:{{node}}", - "metric": "network", - "refId": "A", - "step": 10 - }, - { - "exemplar": true, - "expr": "sum (irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\"}[2m]))by (node) *8", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Outflows:{{node}}", - "metric": "network", - "refId": "B", - "step": 10 - } - ], - "title": "$Node：Node Network Overview", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "auto" - }, - "filterable": false, - "inspect": false - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "CPU Limitations" - }, - "properties": [ - { - "id": "custom.width", - "value": 76 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 71 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Limits" - }, - "properties": [ - { - "id": "custom.width", - "value": 74 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Disk Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 74 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*%" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "color-background" - } - }, - { - "id": "color", - "value": { - "mode": "continuous-GrYlRd" - } - }, - { - "id": "custom.width", - "value": 85 - }, - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "decimals", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(Memory Usage|Total Memory|Memory Request|Memory Limit|Disk Usage|Disk Total)" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Nodes" - }, - "properties": [ - { - "id": "custom.width", - "value": 96 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Requests" - }, - "properties": [ - { - "id": "custom.width", - "value": 76 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "CPU Requests" - }, - "properties": [ - { - "id": "custom.width", - "value": 75 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(CPUTotal|MemoryTotal|DiskTotal|PodLimit)" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "color-background" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "blue" - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PodCap" - }, - "properties": [ - { - "id": "custom.width", - "value": 66 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "CPU Core Usage$|Memory Usage$|Disk Usage$|Pod Number of" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Total/" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "color-background" - } - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "color-background" - } - }, - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - }, - { - "id": "decimals", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Pod Number of" - }, - "properties": [ - { - "id": "custom.width", - "value": 58 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "CPUTotal Cores" - }, - "properties": [ - { - "id": "custom.width", - "value": 69 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Memory" - }, - "properties": [ - { - "id": "custom.width", - "value": 75 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Disk" - }, - "properties": [ - { - "id": "custom.width", - "value": 74 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "CPU Core Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 74 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Usage%" - }, - "properties": [ - { - "id": "custom.width", - "value": 102 - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 52, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Memory Usage%" - } - ] - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "exemplar": false, - "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",created_by_kind!~\"|Job\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "pod Number", - "refId": "A" - }, - { - "exemplar": false, - "expr": "kube_node_status_condition{origin_prometheus=~\"$origin_prometheus\",status=\"true\",node=~\"^$Node$\"} == 1", - "format": "table", - "hide": true, - "instant": true, - "interval": "", - "legendFormat": "Status", - "refId": "B" - }, - { - "exemplar": false, - "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m])) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "I" - }, - { - "exemplar": false, - "expr": "kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"} - 0", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "C" - }, - { - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "E" - }, - { - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "F" - }, - { - "exemplar": false, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "J" - }, - { - "exemplar": false, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"}) by (node) - 0", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "D" - }, - { - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "G" - }, - { - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "H" - }, - { - "exemplar": false, - "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "K" - }, - { - "exemplar": false, - "expr": "sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "L" - }, - { - "exemplar": false, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Memory Usage%", - "refId": "M" - }, - { - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Memory Requests%", - "refId": "N" - }, - { - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Memory Limit%", - "refId": "O" - }, - { - "exemplar": false, - "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m]))by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "CPU Usage%", - "refId": "P" - }, - { - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "CPU Requests%", - "refId": "Q" - }, - { - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Memory Limit%", - "refId": "R" - }, - { - "exemplar": false, - "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})by (node) / sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Disk Usage%", - "refId": "S" - }, - { - "exemplar": false, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"pods\", unit=\"integer\",node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "PodCaps", - "refId": "T" - } - ], - "title": "$Node：Node Information Detail", - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 10": true, - "Time 11": true, - "Time 12": true, - "Time 13": true, - "Time 14": true, - "Time 15": true, - "Time 16": true, - "Time 17": true, - "Time 18": true, - "Time 19": true, - "Time 2": true, - "Time 20": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "Time 6": true, - "Time 7": true, - "Time 8": true, - "Time 9": true, - "Value #B": true, - "Value #E": false, - "Value #F": false, - "__name__": true, - "app_kubernetes_io_name": true, - "app_kubernetes_io_name 1": true, - "app_kubernetes_io_name 2": true, - "app_kubernetes_io_name 3": true, - "app_kubernetes_io_version": true, - "app_kubernetes_io_version 1": true, - "app_kubernetes_io_version 2": true, - "app_kubernetes_io_version 3": true, - "condition": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "instance 3": true, - "job": true, - "job 1": true, - "job 2": true, - "job 3": true, - "k8s_namespace": true, - "k8s_namespace 1": true, - "k8s_namespace 2": true, - "k8s_namespace 3": true, - "k8s_sname": true, - "k8s_sname 1": true, - "k8s_sname 2": true, - "k8s_sname 3": true, - "origin_prometheus": true, - "origin_prometheus 1": true, - "origin_prometheus 2": true, - "origin_prometheus 3": true, - "resource": true, - "status": true, - "unit": true - }, - "includeByName": {}, - "indexByName": { - "Time": 22, - "Value #A": 2, - "Value #C": 6, - "Value #D": 8, - "Value #E": 16, - "Value #F": 17, - "Value #G": 18, - "Value #H": 19, - "Value #I": 7, - "Value #J": 9, - "Value #K": 11, - "Value #L": 10, - "Value #M": 4, - "Value #N": 13, - "Value #O": 15, - "Value #P": 3, - "Value #Q": 12, - "Value #R": 14, - "Value #S": 5, - "Value #T": 1, - "instance": 23, - "job": 24, - "node": 0, - "origin_prometheus": 25, - "resource": 20, - "unit": 21 - }, - "renameByName": { - "Value #A": "Pod Number of", - "Value #C": "CPU Total Cores", - "Value #D": "Total Memory", - "Value #E": "CPU Requests", - "Value #F": "CPU Limit", - "Value #G": "Memory Requests", - "Value #H": "Memory Limit", - "Value #I": "CPU Core Usage", - "Value #J": "Memory Usage", - "Value #K": "Disk Usage", - "Value #L": "Disk Total", - "Value #M": "Memory Usage%", - "Value #N": "Memory Requests%", - "Value #O": "Memory Limit%", - "Value #P": "CPU Usage%", - "Value #Q": "CPU Requests%", - "Value #R": "CPU Limit%", - "Value #S": "Disk Usage%", - "Value #T": "Pod Limit", - "condition": "Status", - "node": "Node" - } - } - }, - { - "id": "filterFieldsByName", - "options": {} - } - ], - "type": "table" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "color-background" - }, - "inspect": false - }, - "decimals": 0, - "links": [], - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 0 - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "red", - "index": 1 - } - }, - "type": "special" - } - ], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Utilization" - }, - "properties": [ - { - "id": "unit", - "value": "percent" - }, - { - "id": "custom.width", - "value": 54 - }, - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - }, - { - "id": "decimals" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "from": 75, - "result": { - "color": "semi-dark-red", - "index": 0 - }, - "to": 110 - }, - "type": "range" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mounts Pod Number of" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "custom.width", - "value": 59 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Namespace" - }, - "properties": [ - { - "id": "custom.width", - "value": 58 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PVC" - }, - "properties": [ - { - "id": "custom.width", - "value": 94 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 57 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "custom.width", - "value": 54 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 33 - }, - "id": 92, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Usage Rate" - } - ] - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "exemplar": false, - "expr": "max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"})", - "format": "table", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{namespace}}:{{ persistentvolumeclaim }}", - "metric": "container_memory_usage:sort_desc", - "range": false, - "refId": "A", - "step": 10 - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "min by (namespace,persistentvolumeclaim) (kubelet_volume_stats_available_bytes{origin_prometheus=~\"$origin_prometheus\"}) + max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "metric": "container_memory_usage:sort_desc", - "range": false, - "refId": "B", - "step": 10 - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"}) /(min by (namespace,persistentvolumeclaim) (kubelet_volume_stats_available_bytes{origin_prometheus=~\"$origin_prometheus\"}) + max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"}))*100", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{namespace}}:{{ persistentvolumeclaim }}", - "metric": "container_memory_usage:sort_desc", - "range": false, - "refId": "C", - "step": 10 - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "count by (namespace,persistentvolumeclaim)(kube_pod_spec_volumes_persistentvolumeclaims_info{origin_prometheus=~\"$origin_prometheus\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "metric": "container_memory_usage:sort_desc", - "range": false, - "refId": "D", - "step": 10 - } - ], - "title": "PVC Storage Usage", - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true - }, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "Value #A": "Usage", - "Value #B": "Total", - "Value #C": "Usage Rate", - "Value #D": "Mount Pod Number", - "namespace": "Namespaces", - "persistentvolumeclaim": "PVC" - } - } - } - ], - "type": "table" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 9, - "x": 6, - "y": 33 - }, - "id": 86, - "options": { - "legend": { - "calcs": ["last", "max"], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container !=\"\",container!=\"POD\"}[2m])) by (namespace)>0.5", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ namespace }}", - "metric": "container_cpu", - "refId": "A", - "step": 10 - } - ], - "title": "Namespaces CPU Usage kernel(>0.5)", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 9, - "x": 15, - "y": 33 - }, - "id": 85, - "options": { - "legend": { - "calcs": ["last", "max"], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container !=\"\",container!=\"POD\"}) by (namespace) > 1*1024*1024*1024", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{namespace} {{ pod }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "A", - "step": 10 - } - ], - "title": "Namespaces WSS Memory Usage (>1G)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 41 - }, - "id": 49, - "panels": [], - "title": "Pod Resource Overview：SelectedPod:【$Pod】", - "type": "row" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "auto" - }, - "filterable": false, - "inspect": false - }, - "displayName": "", - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green" - }, - { - "color": "#EAB839", - "value": 80 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Namespace" - }, - "properties": [ - { - "id": "custom.width", - "value": 96 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Pod Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 207 - }, - { - "id": "custom.align", - "value": "right" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cores Used" - }, - "properties": [ - { - "id": "custom.width", - "value": 71 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Reboot" - }, - "properties": [ - { - "id": "custom.width", - "value": 38 - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "#EAB839", - "value": 1 - }, - { - "color": "red", - "value": 3 - } - ] - } - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "color-background" - } - }, - { - "id": "decimals" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*%" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "color", - "value": { - "mode": "continuous-GrYlRd" - } - }, - { - "id": "decimals", - "value": 1 - }, - { - "id": "custom.width", - "value": 55 - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 1 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Restrictions" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "color-background" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Nodes" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "Cores in use$|WSS$|RSS$|Survival|Inflow|Outflow" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Container Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 57 - }, - { - "id": "custom.align", - "value": "left" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Survival" - }, - "properties": [ - { - "id": "unit", - "value": "s" - }, - { - "id": "custom.width", - "value": 80 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Used cores" - }, - "properties": [ - { - "id": "custom.width", - "value": 62 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "CPU Limits" - }, - "properties": [ - { - "id": "custom.width", - "value": 58 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Limit" - }, - "properties": [ - { - "id": "custom.width", - "value": 68 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Requirements" - }, - "properties": [ - { - "id": "custom.width", - "value": 88 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "WSS$|RSS$|Memory Requirements$|Memory Limit$|Disk.*$" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "WSS" - }, - "properties": [ - { - "id": "custom.width", - "value": 81 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RSS" - }, - "properties": [ - { - "id": "custom.width", - "value": 74 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "CPU Requirements" - }, - "properties": [ - { - "id": "custom.width", - "value": 72 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Disk Limitations" - }, - "properties": [ - { - "id": "custom.width", - "value": 83 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Disk Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 72 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "orange", - "value": 10737418240 - }, - { - "color": "red", - "value": 16106127360 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "WSS%" - }, - "properties": [ - { - "id": "custom.width", - "value": 77 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/Inflow|Streaming Out/" - }, - "properties": [ - { - "id": "unit", - "value": "binbps" - }, - { - "id": "custom.width", - "value": 80 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 42 - }, - "id": 47, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Container Name" - } - ] - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "exemplar": false, - "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod,node,namespace) / (sum(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}/100000) by (container, pod,node,namespace)) ", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "exemplar": false, - "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "CPUKernel Usage", - "refId": "Q" - }, - { - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "B" - }, - { - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "C" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "wss%", - "refId": "I" - }, - { - "exemplar": false, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "wss", - "refId": "D" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "rss%", - "refId": "L" - }, - { - "exemplar": false, - "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "rss", - "refId": "K" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "E" - }, - { - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "F" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "sum(container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "J" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "kube_pod_container_status_restarts_total{\n origin_prometheus=~\"$origin_prometheus\",\n pod=~\"$Pod\",\n container=~\"$Container\",\n container!=\"\",\n container!=\"POD\",\n namespace=~\"$NameSpace\"\n}\n* on (pod) group_left(node) (\n kube_pod_info{\n origin_prometheus=~\"$origin_prometheus\",\n pod=~\"$Pod\",\n node=~\"^$Node$\",\n namespace=~\"$NameSpace\"\n }\n unless on (pod) (\n kube_pod_info{\n origin_prometheus=~\"$origin_prometheus\",\n pod=~\"$Pod\",\n node=~\"^$Node$\",\n namespace=~\"$NameSpace\"\n } offset 1m\n )\n)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "H" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "time() - \n(\n max by(pod, namespace) (kube_pod_created{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\"})\n * on(pod, namespace) group_right(container)\n kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container=~\"$Container\"}\n)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "__auto", - "refId": "R" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "sum(container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "S" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "sum(sum(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "__auto", - "refId": "T" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "sum(sum(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "__auto", - "refId": "U" - } - ], - "title": "$Node：Pod Resource Detail(Associable Nodes)", - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 10": true, - "Time 11": true, - "Time 12": true, - "Time 13": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "Time 6": true, - "Time 7": true, - "Time 8": true, - "Time 9": true, - "Value #G": false, - "__name__": true, - "app_kubernetes_io_name": true, - "app_kubernetes_io_name 1": true, - "app_kubernetes_io_name 2": true, - "app_kubernetes_io_version": true, - "app_kubernetes_io_version 1": true, - "app_kubernetes_io_version 2": true, - "container 1": true, - "container 10": true, - "container 11": true, - "container 12": true, - "container 2": true, - "container 3": true, - "container 4": true, - "container 5": true, - "container 6": true, - "container 7": true, - "container 8": true, - "container 9": true, - "created_by_kind": true, - "created_by_name": true, - "host_ip": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "job": true, - "job 1": true, - "job 2": true, - "k8s_namespace": true, - "k8s_namespace 1": true, - "k8s_namespace 2": true, - "k8s_sname": true, - "k8s_sname 1": true, - "k8s_sname 2": true, - "namespace": false, - "namespace 1": true, - "namespace 10": true, - "namespace 11": true, - "namespace 12": false, - "namespace 2": true, - "namespace 3": true, - "namespace 4": true, - "namespace 5": true, - "namespace 6": true, - "namespace 7": true, - "namespace 8": true, - "namespace 9": true, - "node 1": true, - "node 10": true, - "node 11": false, - "node 12": true, - "node 2": true, - "node 3": true, - "node 4": true, - "node 5": true, - "node 6": true, - "node 7": true, - "node 8": true, - "node 9": true, - "origin_prometheus": true, - "origin_prometheus 1": true, - "origin_prometheus 2": true, - "phase": true, - "pod_ip": true, - "priority_class": true, - "uid": true - }, - "includeByName": {}, - "indexByName": { - "Time": 21, - "Value #A": 4, - "Value #B": 16, - "Value #C": 7, - "Value #D": 10, - "Value #E": 17, - "Value #F": 9, - "Value #G": 23, - "Value #H": 14, - "Value #I": 5, - "Value #J": 13, - "Value #K": 11, - "Value #L": 6, - "Value #M": 24, - "Value #N": 25, - "Value #O": 26, - "Value #P": 27, - "Value #Q": 8, - "Value #R": 15, - "Value #S": 12, - "container": 2, - "instance": 18, - "ip": 28, - "job": 19, - "namespace": 1, - "node": 0, - "origin_prometheus": 20, - "pod": 3, - "uid": 22 - }, - "renameByName": { - "Value #A": "CPU%", - "Value #B": "CPU Requirements", - "Value #C": "CPU Limitations", - "Value #D": "WSS", - "Value #E": "Memory Requirements", - "Value #F": "Memory Limit", - "Value #H": "Reboot", - "Value #I": "WSS%", - "Value #J": "Disk Usage", - "Value #K": "RSS", - "Value #L": "RSS%", - "Value #M": "Heap Memory", - "Value #N": "maxHeap", - "Value #O": "Non-Heap", - "Value #P": "maxNon-Heap", - "Value #Q": "Using Cores", - "Value #R": "Survival", - "Value #S": "Disk Limit", - "Value #T": "Inflow", - "Value #U": "Streaming Out", - "container": "Container Name", - "instance": "", - "ip": "POD IP", - "namespace": "Namespace", - "namespace 1": "", - "namespace 12": "Namespace", - "node": "Nodes", - "node 1": "", - "node 11": "Node", - "pod": "Pod Name", - "priority_class": "" - } - } - }, - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "Node", - "Namespace", - "Container Name", - "Pod Name", - "CPU%", - "WSS%", - "RSS%", - "CPU Restrictions", - "Cores Used", - "Memory Limit", - "WSS", - "RSS", - "Disk Limit", - "Disk Usage", - "Reboot", - "CPUDemand", - "Memory Requirements", - "Inflow", - "Outflow", - "Survival" - ] - } - } - } - ], - "type": "table" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 50 - }, - "id": 58, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "max(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod) / (max(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}/100000) by (container, pod)) * 100", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ pod }}", - "metric": "container_cpu", - "refId": "A", - "step": 10 - } - ], - "title": "Pod Containers CPU Utilization (Maximum100%Associable Nodes)", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 50 - }, - "id": 27, - "options": { - "legend": { - "calcs": ["max", "last", "mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "exemplar": false, - "expr": "max (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod)/ max(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod) * 100", - "interval": "", - "intervalFactor": 1, - "legendFormat": "WSS：{{ pod }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "A", - "step": 10 - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "max (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod)/ max(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod) * 100", - "interval": "", - "intervalFactor": 1, - "legendFormat": "RSS：{{ pod }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "B", - "step": 10 - }, - { - "editorMode": "code", - "exemplar": true, - "expr": "(cass_jvm_heap{service=~\"$Container\"} * on (pod_ip) group_right(service) kube_pod_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\"}) / (cass_jvm_heap_max{service=~\"$Container\"} * on (pod_ip) group_right(service) kube_pod_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\"}) * 100", - "hide": true, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Heap：{{ pod }}", - "metric": "container_memory_usage:sort_desc", - "refId": "C", - "step": 10 - } - ], - "title": "Pod Container Memory Usage (Associatable Nodes)", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binbps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 50 - }, - "id": 77, - "options": { - "legend": { - "calcs": ["max", "last", "mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "exemplar": false, - "expr": "max(max(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Inflow:{{ pod}}", - "metric": "network", - "range": true, - "refId": "A", - "step": 10 - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "max(max(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Outflow:{{ pod}}", - "metric": "network", - "range": true, - "refId": "B", - "step": 10 - }, - { - "editorMode": "code", - "expr": "sum(sum(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info) by(pod) *8", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "metric": "network", - "range": true, - "refId": "C", - "step": 10 - }, - { - "editorMode": "code", - "expr": "sum(sum(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info) by(pod) *8", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "metric": "network", - "range": true, - "refId": "D", - "step": 10 - } - ], - "title": "Pod Network bandwidth per second (Associable Nodes)", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Limit.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 59 - }, - "id": 82, - "options": { - "legend": { - "calcs": ["max", "last", "mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "max(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod,node,namespace)", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU Usage：{{ pod }}", - "metric": "container_cpu", - "refId": "A", - "step": 10 - }, - { - "editorMode": "code", - "exemplar": true, - "expr": "max(max(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)) by(container)", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Pod CPU Limitations：{{ container}}", - "metric": "container_cpu", - "refId": "B", - "step": 10 - } - ], - "title": "Pod Containers CPU Core Usage", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Limitations/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 59 - }, - "id": 84, - "options": { - "legend": { - "calcs": ["max", "last", "mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "exemplar": false, - "expr": "max (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "WSS：{{ pod }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "A", - "step": 10 - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "max(max(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)) by(container)", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "PodMemory Limits：{{ container}}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "B", - "step": 10 - } - ], - "title": "Pod Containers WSS Memory Usage (Associable Nodes)", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 59 - }, - "id": 83, - "options": { - "legend": { - "calcs": ["lastNotNull", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "max (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "RSS：{{ pod }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "A", - "step": 10 - } - ], - "title": "Pod Containers RSS Memory Usage (Associable Nodes)", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 68 - }, - "id": 61, - "panels": [ - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "auto" - }, - "filterable": false, - "inspect": false - }, - "displayName": "", - "mappings": [], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*%" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge", - "valueDisplayMode": "color" - } - }, - { - "id": "max", - "value": 1 - }, - { - "id": "min", - "value": 0 - }, - { - "id": "color", - "value": { - "mode": "continuous-GrYlRd" - } - }, - { - "id": "decimals", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Memory Usage$|.*Memory Limits$|.*Memory Requirements$|.*Disk Usage$|.*Disk Limits$" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Namespace" - }, - "properties": [ - { - "id": "custom.width", - "value": 92 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Container Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 187 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total CPU Core Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Pod" - }, - "properties": [ - { - "id": "custom.width", - "value": 44 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Average CPU Usage%" - }, - "properties": [ - { - "id": "custom.width", - "value": 116 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Average RSS Memory Usage%" - }, - "properties": [ - { - "id": "custom.width", - "value": 141 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Average WSS Memory Usage%" - }, - "properties": [ - { - "id": "custom.width", - "value": 165 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total CPU Limit" - }, - "properties": [ - { - "id": "custom.width", - "value": 86 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Memory Limit" - }, - "properties": [ - { - "id": "custom.width", - "value": 86 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Limit$/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Memory Usage$|.*Core Usage$/" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RSS Memory Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 107 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total WSS Memory Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 113 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Average Disk Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 96 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "orange", - "value": 10737418240 - }, - { - "color": "red", - "value": 16106127360 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Average Disk Limit" - }, - "properties": [ - { - "id": "custom.width", - "value": 96 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total CPU Demand" - }, - "properties": [ - { - "id": "custom.width", - "value": 80 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Memory Demand" - }, - "properties": [ - { - "id": "custom.width", - "value": 80 - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 87, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Average WSS Memory Usage%" - } - ] - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "editorMode": "code", - "exemplar": false, - "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container) / (sum(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}/100000) by (container))", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Total Cores Used", - "refId": "L" - }, - { - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "B" - }, - { - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "C" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "I" - }, - { - "exemplar": false, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "D" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Average Memory%(RSS)", - "refId": "H" - }, - { - "exemplar": false, - "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Total Memory Usage (RSS) ", - "refId": "K" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "sum by(container) (kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "E" - }, - { - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "F" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "avg(container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "J" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "count(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by(container,namespace)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "G" - }, - { - "editorMode": "code", - "exemplar": false, - "expr": "avg(container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "M" - } - ], - "title": "Microservices (Container Name) Resource Statistics", - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 10": true, - "Time 11": true, - "Time 12": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "Time 6": true, - "Time 7": true, - "Time 8": true, - "Time 9": true - }, - "includeByName": {}, - "indexByName": { - "Time": 15, - "Value #A": 3, - "Value #B": 13, - "Value #C": 6, - "Value #D": 9, - "Value #E": 14, - "Value #F": 8, - "Value #G": 2, - "Value #H": 5, - "Value #I": 4, - "Value #J": 12, - "Value #K": 10, - "Value #L": 7, - "Value #M": 11, - "container": 1, - "namespace": 0 - }, - "renameByName": { - "Time 1": "", - "Value #A": "Average CPU Usage%", - "Value #B": "Total CPU Demand", - "Value #C": "Total CPU Restrictions", - "Value #D": "Total WSS Memory Usage", - "Value #E": "Total Memory Requirements", - "Value #F": "Total Memory Limit", - "Value #G": "Pod", - "Value #H": "Average RSS Memory Usage%", - "Value #I": "Average WSS Memory Usage%", - "Value #J": "Average Disk Usage", - "Value #K": "Total RSS Memory Use", - "Value #L": "Total CPU Core Usage", - "Value #M": "Average Disk Limit", - "container": "Container Name", - "namespace": "Namespace" - } - } - }, - { - "id": "filterFieldsByName", - "options": {} - } - ], - "type": "table" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 12 - }, - "id": 24, - "options": { - "legend": { - "calcs": ["max", "last", "mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "exemplar": true, - "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container) / (sum(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}/100000) by (container)) * 100", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ container}}", - "metric": "container_cpu", - "refId": "A", - "step": 10 - } - ], - "title": "Microservices (Container Name) Average CPU Usage (Maximum100%)", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 12 - }, - "id": 89, - "options": { - "legend": { - "calcs": ["max", "last", "mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container) * 100", - "interval": "", - "intervalFactor": 1, - "legendFormat": "WSS：{{ container }}", - "metric": "container_memory_usage:sort_desc", - "refId": "A", - "step": 10 - }, - { - "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container) * 100", - "interval": "", - "intervalFactor": 1, - "legendFormat": "RSS：{{ container }}", - "metric": "container_memory_usage:sort_desc", - "refId": "B", - "step": 10 - } - ], - "title": "Microservice (Container Name) Average Memory Utilization", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binbps" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 12 - }, - "id": 16, - "options": { - "legend": { - "calcs": ["mean", "last", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "sum(sum(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(container) *8", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Inflow:{{ container }}", - "metric": "network", - "range": true, - "refId": "A", - "step": 10 - }, - { - "editorMode": "code", - "exemplar": true, - "expr": "sum(sum(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(container) *8", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Outflow:{{ container }}", - "metric": "network", - "range": true, - "refId": "B", - "step": 10 - }, - { - "expr": "sum (rate (container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "-> {{ pod }}", - "metric": "network", - "refId": "C", - "step": 10 - }, - { - "expr": "- sum (rate (container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "<- {{ pod }}", - "metric": "network", - "refId": "D", - "step": 10 - } - ], - "title": "Microservice (Container Name) Network bandwidth per second (Associable Nodes)", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/CPU Limit.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 91, - "options": { - "legend": { - "calcs": ["max", "last", "mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "editorMode": "code", - "exemplar": true, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU Limitations：{{ container}}", - "metric": "container_cpu", - "refId": "A", - "step": 10 - }, - { - "editorMode": "code", - "exemplar": true, - "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container)", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU Core Usage：{{ container}}", - "metric": "container_cpu", - "refId": "B", - "step": 10 - } - ], - "title": "Microservices (Container Name) Overall CPU Cores Used", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Memory Limit.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 90, - "options": { - "legend": { - "calcs": ["max", "last", "mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "WSS：{{ container }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "A", - "step": 10 - }, - { - "editorMode": "code", - "expr": "sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Memory Limit：{{ container }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "B", - "step": 10 - }, - { - "editorMode": "code", - "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "RSS：{{ container }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "C", - "step": 10 - } - ], - "title": "Microservices (Container Name) Overall Memory Usage", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 59, - "options": { - "legend": { - "calcs": ["max", "last", "mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "editorMode": "code", - "expr": "count(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by(container,namespace)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{namespace}}：{{ container }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "A", - "step": 10 - } - ], - "title": "Microservice (Container Name) Pod Number", - "type": "timeseries" - } - ], - "title": "Microservices (Container Name) Resource Overview：Selected Microservices:【$Container】", - "type": "row" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 41, - "tags": ["Prometheus", "Kubernetes"], - "templating": { - "list": [ - { - "current": { - "text": "", - "value": "" - }, - "definition": "label_values(kube_node_info,origin_prometheus)", - "includeAll": false, - "label": "K8S", - "name": "origin_prometheus", - "options": [], - "query": { - "query": "label_values(kube_node_info,origin_prometheus)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 5, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "label_values(kube_node_info{origin_prometheus=~\"$origin_prometheus\"},node)", - "includeAll": true, - "label": "Nodes", - "name": "Node", - "options": [], - "query": { - "query": "label_values(kube_node_info{origin_prometheus=~\"$origin_prometheus\"},node)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 5, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "label_values(kube_namespace_created{origin_prometheus=~\"$origin_prometheus\"},namespace)", - "includeAll": true, - "label": "Namespace", - "name": "NameSpace", - "options": [], - "query": { - "query": "label_values(kube_namespace_created{origin_prometheus=~\"$origin_prometheus\"},namespace)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 5, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\"},container)", - "includeAll": true, - "label": "Microservice (Container Name)", - "name": "Container", - "options": [], - "query": { - "query": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\"},container)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 5, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container=~\"$Container\"},pod)", - "includeAll": true, - "label": "Pod", - "name": "Pod", - "options": [], - "query": { - "query": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container=~\"$Container\"},pod)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 5, - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": ["30s", "1m", "5m", "15m", "30m", "1h"] - }, - "timezone": "", - "title": "K8S Dashboard EN 20250125", - "uid": "15661", - "version": 3 - }, - "overwrite": true -} diff --git a/rootfs/usr/share/grafana/api/dashboards/kubernetes_apiserver.json b/rootfs/usr/share/grafana/api/dashboards/kubernetes_apiserver.json deleted file mode 100644 index 5f5ae6f..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/kubernetes_apiserver.json +++ /dev/null @@ -1,642 +0,0 @@ -{ - "dashboard": { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": 12006, - "graphTooltip": 0, - "id": null, - "iteration": 1585691077160, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(apiserver_request_total[5m])) by (verb)", - "legendFormat": "{{verb}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "apiserver request rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "req/s", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(workqueue_queue_duration_seconds_bucket[5m])) by (le))", - "legendFormat": "p95", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket[5m])) by (le))", - "legendFormat": "p90", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.5, sum(rate(workqueue_queue_duration_seconds_bucket[5m])) by (le))", - "legendFormat": "p50", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "workqueue service time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": "service time", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "95th percentile request latency for Kubernetes apiserver component", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 8 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\"CONNECT|WATCH\"}[5m])) by (le))", - "legendFormat": "p95", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.90, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\"CONNECT|WATCH\"}[5m])) by (le))", - "legendFormat": "p90", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\"CONNECT|WATCH\"}[5m])) by (le))", - "legendFormat": "p50 ", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "apiserver request latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "s", - "label": "latency", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(workqueue_work_duration_seconds_bucket[5m])) by (le))", - "legendFormat": "p95", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.9, sum(rate(workqueue_work_duration_seconds_bucket[5m])) by (le))", - "legendFormat": "p90", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.5, sum(rate(workqueue_work_duration_seconds_bucket[5m])) by (le))", - "legendFormat": "p50", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "workqueue processing time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": "processing time", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "hiddenSeries": false, - "id": 6, - "interval": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(etcd_helper_cache_hit_total[5m]))/sum((rate(etcd_helper_cache_hit_total[5m])+rate(etcd_helper_cache_miss_total[5m])))", - "legendFormat": "ratio", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "etcd helper cache hit ratio", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "percentunit", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "etcd request latency (p95)", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 17 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))", - "legendFormat": "p95", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.90, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))", - "legendFormat": "p90", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.5, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))", - "legendFormat": "p50", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "etcd request latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "s", - "label": "latency", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": false, - "schemaVersion": 22, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "includeAll": false, - "label": "Data source", - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Kubernetes apiserver", - "uid": "R6abPf9Zz", - "version": 3, - "description": "Latency and cache hit graphs for Kubernetes apiserver. Useful for observing performance of Amazon EKS clusters." - }, - "overwrite": true -} diff --git a/rootfs/usr/share/grafana/api/dashboards/kubernetes_cluster.json b/rootfs/usr/share/grafana/api/dashboards/kubernetes_cluster.json deleted file mode 100644 index c402452..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/kubernetes_cluster.json +++ /dev/null @@ -1,2013 +0,0 @@ -{ - "dashboard": { - "id": null, - "title": "Kubernetes cluster monitoring (via Prometheus)", - "description": "Monitors Kubernetes cluster using Prometheus. Shows overall cluster CPU / Memory / Filesystem usage as well as individual pod, containers, systemd services statistics. Uses cAdvisor metrics only.", - "tags": ["kubernetes"], - "style": "dark", - "timezone": "browser", - "editable": true, - "hideControls": false, - "sharedCrosshair": false, - "rows": [ - { - "collapse": false, - "editable": true, - "height": "200px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "decimals": 2, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)", - "thresholdLine": false - }, - "height": "200px", - "id": 32, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": 200, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum (rate (container_network_receive_bytes_total{kubernetes_io_hostname=~\"^$Node$\"}[1m]))", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "Received", - "metric": "network", - "refId": "A", - "step": 10 - }, - { - "expr": "- sum (rate (container_network_transmit_bytes_total{kubernetes_io_hostname=~\"^$Node$\"}[1m]))", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "Sent", - "metric": "network", - "refId": "B", - "step": 10 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Network I/O pressure", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "transparent": false, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "title": "Network I/O pressure" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "editable": true, - "error": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "height": "180px", - "id": 4, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "sum (container_memory_working_set_bytes{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) / sum (machine_memory_bytes{kubernetes_io_hostname=~\"^$Node$\"}) * 100", - "interval": "10s", - "intervalFactor": 1, - "refId": "A", - "step": 10 - } - ], - "thresholds": "65, 90", - "title": "Cluster memory usage", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "decimals": 2, - "editable": true, - "error": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "height": "180px", - "id": 6, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "sum (rate (container_cpu_usage_seconds_total{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) / sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"}) * 100", - "interval": "10s", - "intervalFactor": 1, - "refId": "A", - "step": 10 - } - ], - "thresholds": "65, 90", - "title": "Cluster CPU usage (1m avg)", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "decimals": 2, - "editable": true, - "error": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "height": "180px", - "id": 7, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "sum (container_fs_usage_bytes{device=~\"^/dev/.*$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) / sum (container_fs_limit_bytes{device=~\"^/dev/.*$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) * 100", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": "65, 90", - "title": "Cluster filesystem usage", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "decimals": 2, - "editable": true, - "error": false, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "height": "1px", - "id": 9, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "20%", - "prefix": "", - "prefixFontSize": "20%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "sum (container_memory_working_set_bytes{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"})", - "interval": "10s", - "intervalFactor": 1, - "refId": "A", - "step": 10 - } - ], - "thresholds": "", - "title": "Used", - "type": "singlestat", - "valueFontSize": "50%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "decimals": 2, - "editable": true, - "error": false, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "height": "1px", - "id": 10, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "sum (machine_memory_bytes{kubernetes_io_hostname=~\"^$Node$\"})", - "interval": "10s", - "intervalFactor": 1, - "refId": "A", - "step": 10 - } - ], - "thresholds": "", - "title": "Total", - "type": "singlestat", - "valueFontSize": "50%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "decimals": 2, - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "height": "1px", - "id": 11, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": " cores", - "postfixFontSize": "30%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "sum (rate (container_cpu_usage_seconds_total{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m]))", - "interval": "10s", - "intervalFactor": 1, - "refId": "A", - "step": 10 - } - ], - "thresholds": "", - "title": "Used", - "type": "singlestat", - "valueFontSize": "50%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "decimals": 2, - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "height": "1px", - "id": 12, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": " cores", - "postfixFontSize": "30%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"})", - "interval": "10s", - "intervalFactor": 1, - "refId": "A", - "step": 10 - } - ], - "thresholds": "", - "title": "Total", - "type": "singlestat", - "valueFontSize": "50%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "decimals": 2, - "editable": true, - "error": false, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "height": "1px", - "id": 13, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "sum (container_fs_usage_bytes{device=~\"^/dev/.*$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"})", - "interval": "10s", - "intervalFactor": 1, - "refId": "A", - "step": 10 - } - ], - "thresholds": "", - "title": "Used", - "type": "singlestat", - "valueFontSize": "50%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "decimals": 2, - "editable": true, - "error": false, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "height": "1px", - "id": 14, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "sum (container_fs_limit_bytes{device=~\"^/dev/.*$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"})", - "interval": "10s", - "intervalFactor": 1, - "refId": "A", - "step": 10 - } - ], - "thresholds": "", - "title": "Total", - "type": "singlestat", - "valueFontSize": "50%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - } - ], - "showTitle": false, - "title": "Total usage" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "decimals": 3, - "editable": true, - "error": false, - "fill": 0, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "height": "", - "id": 17, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (pod_name)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{ pod_name }}", - "metric": "container_cpu", - "refId": "A", - "step": 10 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pods CPU usage (1m avg)", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "transparent": false, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "none", - "label": "cores", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "showTitle": false, - "title": "Pods CPU usage" - }, - { - "collapse": true, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "decimals": 3, - "editable": true, - "error": false, - "fill": 0, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "height": "", - "id": 23, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "sum (rate (container_cpu_usage_seconds_total{systemd_service_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (systemd_service_name)", - "hide": false, - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{ systemd_service_name }}", - "metric": "container_cpu", - "refId": "A", - "step": 10 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "System services CPU usage (1m avg)", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "none", - "label": "cores", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "title": "System services CPU usage" - }, - { - "collapse": true, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "decimals": 3, - "editable": true, - "error": false, - "fill": 0, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "height": "", - "id": 24, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",name=~\"^k8s_.*\",container_name!=\"POD\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (container_name, pod_name)", - "hide": false, - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "pod: {{ pod_name }} | {{ container_name }}", - "metric": "container_cpu", - "refId": "A", - "step": 10 - }, - { - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, name, image)", - "hide": false, - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", - "metric": "container_cpu", - "refId": "B", - "step": 10 - }, - { - "expr": "sum (rate (container_cpu_usage_seconds_total{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, rkt_container_name)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", - "metric": "container_cpu", - "refId": "C", - "step": 10 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Containers CPU usage (1m avg)", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "none", - "label": "cores", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "title": "Containers CPU usage" - }, - { - "collapse": true, - "editable": true, - "height": "500px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "decimals": 3, - "editable": true, - "error": false, - "fill": 0, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 20, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "sum (rate (container_cpu_usage_seconds_total{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (id)", - "hide": false, - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{ id }}", - "metric": "container_cpu", - "refId": "A", - "step": 10 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "All processes CPU usage (1m avg)", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "none", - "label": "cores", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "showTitle": false, - "title": "All processes CPU usage" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "decimals": 2, - "editable": true, - "error": false, - "fill": 0, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 25, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 200, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "sum (container_memory_working_set_bytes{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}) by (pod_name)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{ pod_name }}", - "metric": "container_memory_usage:sort_desc", - "refId": "A", - "step": 10 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pods memory usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "title": "Pods memory usage" - }, - { - "collapse": true, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "decimals": 2, - "editable": true, - "error": false, - "fill": 0, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 26, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 200, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "sum (container_memory_working_set_bytes{systemd_service_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}) by (systemd_service_name)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{ systemd_service_name }}", - "metric": "container_memory_usage:sort_desc", - "refId": "A", - "step": 10 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "System services memory usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "title": "System services memory usage" - }, - { - "collapse": true, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "decimals": 2, - "editable": true, - "error": false, - "fill": 0, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 27, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 200, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "sum (container_memory_working_set_bytes{image!=\"\",name=~\"^k8s_.*\",container_name!=\"POD\",kubernetes_io_hostname=~\"^$Node$\"}) by (container_name, pod_name)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "pod: {{ pod_name }} | {{ container_name }}", - "metric": "container_memory_usage:sort_desc", - "refId": "A", - "step": 10 - }, - { - "expr": "sum (container_memory_working_set_bytes{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}) by (kubernetes_io_hostname, name, image)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", - "metric": "container_memory_usage:sort_desc", - "refId": "B", - "step": 10 - }, - { - "expr": "sum (container_memory_working_set_bytes{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}) by (kubernetes_io_hostname, rkt_container_name)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", - "metric": "container_memory_usage:sort_desc", - "refId": "C", - "step": 10 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Containers memory usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "title": "Containers memory usage" - }, - { - "collapse": true, - "editable": true, - "height": "500px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "decimals": 2, - "editable": true, - "error": false, - "fill": 0, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 28, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": 200, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "sum (container_memory_working_set_bytes{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) by (id)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{ id }}", - "metric": "container_memory_usage:sort_desc", - "refId": "A", - "step": 10 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "All processes memory usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "title": "All processes memory usage" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "decimals": 2, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 16, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 200, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (pod_name)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "-> {{ pod_name }}", - "metric": "network", - "refId": "A", - "step": 10 - }, - { - "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (pod_name)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "<- {{ pod_name }}", - "metric": "network", - "refId": "B", - "step": 10 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pods network I/O (1m avg)", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "title": "Pods network I/O" - }, - { - "collapse": true, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "decimals": 2, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 30, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 200, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (container_name, pod_name)", - "hide": false, - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "-> pod: {{ pod_name }} | {{ container_name }}", - "metric": "network", - "refId": "B", - "step": 10 - }, - { - "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (container_name, pod_name)", - "hide": false, - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "<- pod: {{ pod_name }} | {{ container_name }}", - "metric": "network", - "refId": "D", - "step": 10 - }, - { - "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, name, image)", - "hide": false, - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "-> docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", - "metric": "network", - "refId": "A", - "step": 10 - }, - { - "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, name, image)", - "hide": false, - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "<- docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", - "metric": "network", - "refId": "C", - "step": 10 - }, - { - "expr": "sum (rate (container_network_transmit_bytes_total{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, rkt_container_name)", - "hide": false, - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "-> rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", - "metric": "network", - "refId": "E", - "step": 10 - }, - { - "expr": "- sum (rate (container_network_transmit_bytes_total{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, rkt_container_name)", - "hide": false, - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "<- rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", - "metric": "network", - "refId": "F", - "step": 10 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Containers network I/O (1m avg)", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "title": "Containers network I/O" - }, - { - "collapse": true, - "editable": true, - "height": "500px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "decimals": 2, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 29, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": 200, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum (rate (container_network_receive_bytes_total{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (id)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "-> {{ id }}", - "metric": "network", - "refId": "A", - "step": 10 - }, - { - "expr": "- sum (rate (container_network_transmit_bytes_total{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (id)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "<- {{ id }}", - "metric": "network", - "refId": "B", - "step": 10 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "All processes network I/O (1m avg)", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "title": "All processes network I/O" - } - ], - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] - }, - "templating": { - "list": [ - { - "allValue": ".*", - "current": {}, - "hide": 0, - "includeAll": true, - "multi": false, - "name": "Node", - "options": [], - "query": "label_values(kubernetes_io_hostname)", - "refresh": 1, - "type": "query" - } - ] - }, - "annotations": { - "list": [] - }, - "refresh": "10s", - "schemaVersion": 12, - "version": 13, - "uid": "315", - "links": [], - "gnetId": 1621 - }, - "overwrite": true -} diff --git a/rootfs/usr/share/grafana/api/dashboards/kubernetes_deployment_metrics.json b/rootfs/usr/share/grafana/api/dashboards/kubernetes_deployment_metrics.json deleted file mode 100644 index 18e88d4..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/kubernetes_deployment_metrics.json +++ /dev/null @@ -1,1327 +0,0 @@ -{ - "dashboard": { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "Monitors Kubernetes deployments in cluster using Prometheus. Shows overall cluster CPU / Memory / Network of deployments, replicas in each deployment. Uses Kube state metrics and cAdvisor metrics ", - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 741, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 65 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 4, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}) / sum (machine_memory_bytes{kubernetes_io_hostname=~\"^$Node$\"}) * 100", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "", - "range": true, - "refId": "A", - "step": 10 - } - ], - "title": "Deployment memory usage", - "type": "gauge" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 65 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 6, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}[1m])) / sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"}) * 100", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "", - "range": true, - "refId": "A", - "step": 10 - } - ], - "title": "Deployment CPU usage", - "type": "gauge" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 37, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(kube_deployment_status_replicas_available{deployment=~\"^$Deployment$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}) / sum(kube_deployment_status_replicas{deployment=~\"^$Deployment$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}) * 100", - "intervalFactor": 2, - "range": true, - "refId": "A", - "step": 40 - } - ], - "title": "Replicas", - "type": "gauge" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 5 - }, - "id": 38, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{namespace=\"$Namespace\",pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\"})", - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "range": true, - "refId": "A", - "step": 40 - } - ], - "title": "Used", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 5 - }, - "id": 39, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": null, - "expr": "sum (container_memory_working_set_bytes{kubernetes_io_hostname=~\"^$Node.*$\"})", - "intervalFactor": 2, - "refId": "A", - "step": 40 - } - ], - "title": "Total", - "type": "stat" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 5 - }, - "id": 40, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}[1m]))", - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "range": true, - "refId": "A", - "step": 40 - } - ], - "title": "Used", - "type": "stat" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 5 - }, - "id": 41, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": null, - "expr": "sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"})", - "intervalFactor": 2, - "refId": "A", - "step": 40 - } - ], - "title": "Total", - "type": "stat" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 5 - }, - "id": 42, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(kube_deployment_status_replicas_available{namespace=\"$Namespace\",deployment=~\"^$Deployment$\",pod=~\"^$Statefulset.*$\"})", - "intervalFactor": 2, - "range": true, - "refId": "A", - "step": 40 - } - ], - "title": "Available", - "type": "stat" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 5 - }, - "id": 43, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum(kube_deployment_status_replicas{namespace=\"$Namespace\",deployment=~\"^$Deployment$\",pod=~\"^$Statefulset.*$\"})", - "intervalFactor": 2, - "range": true, - "refId": "A", - "step": 40 - } - ], - "title": "Total", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 3, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 8 - }, - "height": "", - "hiddenSeries": false, - "id": 17, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}[1m])) by (pod)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{ pod }}", - "metric": "container_cpu", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Pod CPU usage", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:677", - "format": "none", - "label": "cores", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:678", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 3, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 15 - }, - "height": "", - "hiddenSeries": false, - "id": 24, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{pod!=\"\",pod!=\"POD\",pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}[1m])) by (pod, container)", - "hide": false, - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "container: {{ pod }}:{{ container }} ", - "metric": "container_cpu", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Containers CPU usage", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:189", - "format": "none", - "label": "cores", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:190", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 22 - }, - "hiddenSeries": false, - "id": 25, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 200, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}) by (pod)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{ pod }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Pod memory usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:920", - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:921", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": {}, - "decimals": 2, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 29 - }, - "hiddenSeries": false, - "id": 44, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 200, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": {}, - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}) by (pod, container)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{ pod }}:{{ container }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Container memory usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:920", - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:921", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 2, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 36 - }, - "hiddenSeries": false, - "id": 16, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 200, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": null, - "editorMode": "code", - "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",id!=\"\",pod=~\"^$Statefulset.*$\",namespace=\"${Namespace}\"}[2m])) by (pod)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "-> {{ pod }}", - "metric": "network", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": null, - "editorMode": "code", - "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",id!=\"\",pod=~\"^$statefulset.*$\",namespace=\"${namespace}\"}[2m])) by (pod)", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "<- {{ pod }}", - "metric": "network", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Pod network I/O", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:514", - "format": "Bps", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:515", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - } - ], - "refresh": "10s", - "schemaVersion": 37, - "style": "dark", - "tags": ["kubernetes"], - "templating": { - "list": [ - { - "current": { - "selected": true, - "text": "", - "value": "" - }, - "datasource": null, - "definition": "label_values(namespace)", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "Namespace", - "options": [], - "query": { - "query": "label_values(namespace)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "selected": true, - "text": ["All"], - "value": ["$__all"] - }, - "datasource": {}, - "definition": "label_values({namespace=\"$Namespace\"},deployment)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "Deployment", - "options": [], - "query": { - "query": "label_values({namespace=\"$Namespace\"},deployment)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": true, - "text": ["All"], - "value": ["$__all"] - }, - "definition": "label_values({namespace=\"$Namespace\"},statefulset)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "Statefulset", - "options": [], - "query": { - "query": "label_values({namespace=\"$Namespace\"},statefulset)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": {}, - "definition": "label_values(node)", - "hide": 0, - "includeAll": true, - "multi": false, - "name": "Node", - "options": [], - "query": { - "query": "label_values(node)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] - }, - "timezone": "browser", - "title": "Kubernetes Deployment metrics", - "uid": "wdtm7Mc7z123", - "version": 1, - "weekStart": "" - }, - "overwrite": true -} diff --git a/rootfs/usr/share/grafana/api/dashboards/kubernetes_metrics.json b/rootfs/usr/share/grafana/api/dashboards/kubernetes_metrics.json deleted file mode 100644 index 8607f45..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/kubernetes_metrics.json +++ /dev/null @@ -1,3393 +0,0 @@ -{ - "dashboard": { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Summary metrics about kube-state-metrics v2 version(https://github.com/kubernetes/kube-state-metrics); Referenced 6417", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": false, - "tags": ["kubernetes-app"], - "title": "Dashboards", - "type": "dashboards" - } - ], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 58, - "panels": [], - "title": "Cluster", - "type": "row" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46" - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 80 - }, - { - "color": "#d44a3a", - "value": 90 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "id": 4, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_pod_info{cluster=~\"$cluster\",node=~\"$node\"}) / sum(kube_node_status_allocatable{cluster=~\"$cluster\",resource=\"pods\",node=~\"$node\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Cluster Pod Requested", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46" - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 80 - }, - { - "color": "#d44a3a", - "value": 90 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "id": 5, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_pod_container_resource_requests{cluster=~\"$cluster\",resource=\"cpu\",node=~\"$node\"})/ sum(kube_node_status_allocatable{node=~\"$node\",cluster=~\"$cluster\",resource=\"cpu\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Cluster CPU Requested", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46" - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 80 - }, - { - "color": "#d44a3a", - "value": 90 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "id": 6, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_pod_container_resource_requests{cluster=~\"$cluster\",resource=\"memory\",node=~\"$node\"}) / sum(kube_node_status_allocatable{node=~\"$node\",cluster=~\"$cluster\",resource=\"memory\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Cluster Memory Requested", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "pods", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 5 - }, - "id": 9, - "options": { - "alertThreshold": true, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_node_status_allocatable{cluster=~\"$cluster\",resource=\"pods\",node=~\"$node\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "allocatable", - "refId": "A" - }, - { - "expr": "sum(kube_pod_info{node=~\"$node\",cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "requested", - "refId": "C" - } - ], - "title": "Cluster Pod Capacity", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "cores", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 5 - }, - "id": 10, - "options": { - "alertThreshold": true, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_node_status_capacity{node=~\"$node\",cluster=~\"$cluster\",resource=\"cpu\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "allocatable", - "refId": "A" - }, - { - "expr": "sum(kube_node_status_allocatable{node=~\"$node\",cluster=~\"$cluster\",resource=\"cpu\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "capacity", - "refId": "B" - }, - { - "expr": "sum(kube_pod_container_resource_requests{cluster=~\"$cluster\",resource=\"cpu\",node=~\"$node\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "requested", - "refId": "C" - }, - { - "expr": "sum(kube_pod_container_resource_limits{cluster=~\"$cluster\",resource=\"cpu\",node=~\"$node\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "limited", - "refId": "D" - } - ], - "title": "Cluster CPU Capacity", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bits" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 5 - }, - "id": 11, - "options": { - "alertThreshold": true, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_node_status_allocatable{node=~\"$node\",cluster=~\"$cluster\",resource=\"memory\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "allocatable", - "refId": "A" - }, - { - "expr": "sum(kube_node_status_capacity{node=~\"$node\",cluster=~\"$cluster\",resource=\"memory\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "capacity", - "refId": "B" - }, - { - "expr": "sum(kube_pod_container_resource_requests{cluster=~\"$cluster\",resource=\"memory\",node=~\"$node\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "requested", - "refId": "C" - }, - { - "expr": "sum(kube_pod_container_resource_limits{cluster=~\"$cluster\",resource=\"memory\",node=~\"$node\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "limited", - "refId": "D" - } - ], - "title": "Cluster Mem Capacity", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 22, - "panels": [ - { - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 20 - }, - "id": 24, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_node_info{cluster=~\"$cluster\"})", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Number Of Nodes", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46" - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 1 - }, - { - "color": "#d44a3a" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 20 - }, - "id": 26, - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_node_spec_unschedulable{cluster=~\"$cluster\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Nodes Unavailable", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 8, - "y": 20 - }, - "id": 78, - "options": { - "alertThreshold": true, - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "kube_node_status_condition{cluster=~\"$cluster\",condition=\"Ready\",status=\"false\"}==1", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{node}}", - "refId": "A" - } - ], - "title": "Node NotReady", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 16, - "y": 20 - }, - "id": 51, - "options": { - "alertThreshold": true, - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "kube_node_status_condition{condition=\"DiskPressure\",cluster=~\"$cluster\",status=\"true\"}==1", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{node}} {{condition}}", - "refId": "A" - }, - { - "expr": "kube_node_status_condition{condition=\"MemoryPressure\",node=~\"$node\",cluster=~\"$cluster\",status=\"true\"}==1", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{node}} {{condition}}", - "refId": "B" - }, - { - "expr": "kube_node_status_condition{condition=~\"PIDPressure\",node=~\"$node\",cluster=~\"$cluster\",status=\"true\"}==1", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{node}} {{condition}}", - "refId": "C" - } - ], - "title": "Node Pressure", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 88, - "options": { - "legend": { - "calcs": ["mean", "lastNotNull", "max", "min"], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_node_info{cluster=~\"$cluster\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "node number", - "type": "timeseries" - } - ], - "title": "Node", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 14, - "panels": [ - { - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "displayName", - "value": "Time" - }, - { - "id": "unit", - "value": "time: YYYY-MM-DD HH:mm:ss" - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Metric" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)" - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 0 - }, - {} - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)" - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 1 - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 218 - }, - "id": 16, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "kube_deployment_status_replicas{namespace=~\"$namespace\",cluster=~\"$cluster\"}", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ deployment }}", - "refId": "A" - } - ], - "title": "Deployment Replicas - Up To Date", - "type": "table" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 218 - }, - "id": 18, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_deployment_status_replicas{namespace=~\"$namespace\",cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Deployment Replicas", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 218 - }, - "id": 19, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_deployment_status_replicas_updated{namespace=~\"$namespace\",cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Deployment Replicas - Updated", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 18, - "y": 218 - }, - "id": 20, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_deployment_status_replicas_unavailable{namespace=~\"$namespace\",cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Deployment Replicas - Unavailable", - "type": "stat" - } - ], - "title": "Deployments", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 71, - "panels": [ - { - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "displayName", - "value": "Time" - }, - { - "id": "unit", - "value": "time: YYYY-MM-DD HH:mm:ss" - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Metric" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)" - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 0 - }, - {} - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)" - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 1 - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 22 - }, - "id": 75, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "kube_statefulset_status_replicas_ready{namespace=~\"$namespace\",cluster=~\"$cluster\"}", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ statefulset}}", - "refId": "A" - } - ], - "title": "Statefulset Replicas - Up To Date", - "type": "table" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 6, - "y": 22 - }, - "id": 73, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "kube_statefulset_status_replicas_ready{namespace=~\"$namespace\",cluster=~\"$cluster\"}/kube_statefulset_status_replicas{namespace=~\"$namespace\",cluster=~\"$cluster\"}*100", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{statefulset}}", - "refId": "A" - } - ], - "title": "Statefulset replicas", - "type": "timeseries" - } - ], - "title": "Statefuleset", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 28, - "panels": [ - { - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "decimals": 2, - "displayName": "", - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#56A64B" - }, - { - "color": "#73BF69" - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "displayName", - "value": "Time" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.hidden", - "value": true - }, - { - "id": "custom.align" - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 6, - "x": 0, - "y": 23 - }, - "id": 68, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "changes(kube_pod_container_status_restarts_total{namespace=~\"$namespace\",cluster=~\"$cluster\"}[30m])>1", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Pods restart in 30m", - "type": "table" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 6, - "x": 6, - "y": 23 - }, - "id": 77, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "kube_pod_status_reason{cluster=~\"$cluster\",namespace=~\"$namespace\", reason=\"Evicted\"}==1", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Evicted", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "#629e51", - "mode": "fixed" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 12, - "x": 12, - "y": 23 - }, - "id": 30, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_pod_status_phase{cluster=~\"$cluster\",namespace=~\"$namespace\", phase=\"Running\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Pods Running", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "#629e51", - "mode": "fixed" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 12, - "y": 185 - }, - "id": 33, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_pod_status_phase{cluster=~\"$cluster\",namespace=~\"$namespace\", phase=\"Succeeded\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Pods Succeeded", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "#629e51", - "mode": "fixed" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 18, - "y": 185 - }, - "id": 32, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_pod_status_phase{cluster=~\"$cluster\",namespace=~\"$namespace\", phase=\"Failed\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Pods Failed", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "#629e51", - "mode": "fixed" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 12, - "y": 188 - }, - "id": 31, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_pod_status_phase{cluster=~\"$cluster\",namespace=~\"$namespace\", phase=\"Pending\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Pods Pending", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "#629e51", - "mode": "fixed" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 18, - "y": 188 - }, - "id": 34, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_pod_status_phase{cluster=~\"$cluster\",namespace=~\"$namespace\", phase=\"Unknown\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Pods Unknown", - "type": "stat" - } - ], - "title": "Pods", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 36, - "panels": [ - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 250 - }, - "id": 38, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_pod_container_status_running{namespace=~\"$namespace\",cluster=~\"$cluster\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Containers Running", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 250 - }, - "id": 39, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_pod_container_status_waiting{namespace=~\"$namespace\",cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Containers Waiting", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 12, - "y": 250 - }, - "id": 40, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_pod_container_status_terminated{namespace=~\"$namespace\",cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Containers Terminated", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 18, - "y": 250 - }, - "id": 41, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(changes(kube_pod_container_status_restarts_total{namespace=~\"$namespace\",cluster=~\"$cluster\"}[30m]))", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Containers Restarts (Last 30 Minutes)", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 12, - "x": 0, - "y": 253 - }, - "id": 43, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~\"$namespace\",cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "CPU Cores Requested by Containers", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 12, - "x": 12, - "y": 253 - }, - "id": 42, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~\"$namespace\" ,cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Memory Requested By Containers", - "type": "stat" - } - ], - "title": "Containers", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 80, - "panels": [ - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 82, - "options": { - "alertThreshold": true, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "kube_hpa_status_current_replicas{cluster=~\"$cluster\",namespace=~\"$namespace\"}", - "instant": false, - "interval": "", - "legendFormat": "current_{{hpa}}", - "refId": "A" - }, - { - "expr": "kube_hpa_spec_max_replicas{cluster=~\"$cluster\",namespace=~\"$namespace\"}", - "instant": false, - "interval": "", - "legendFormat": "max_{{hpa}}", - "refId": "B" - }, - { - "expr": "kube_hpa_spec_min_replicas{cluster=~\"$cluster\",namespace=~\"$namespace\"}", - "instant": false, - "interval": "", - "legendFormat": "min_{{hpa}}", - "refId": "C" - } - ], - "title": "hpa", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 108 - }, - "id": 89, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_hpa_status_current_replicas{hpa=~\".*\"})", - "instant": false, - "interval": "", - "legendFormat": "kube_hpa_status_current_replicas", - "refId": "A" - } - ], - "title": "total-hpa-current", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 116 - }, - "id": 90, - "options": {}, - "pluginVersion": "7.1.1", - "targets": [ - { - "expr": "kube_hpa_status_current_replicas{hpa=~\".*\"} == kube_hpa_spec_max_replicas{hpa=~\".*\"}", - "instant": false, - "interval": "", - "legendFormat": "{{hpa}}", - "refId": "A" - } - ], - "title": "current==max", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 116 - }, - "id": 92, - "options": {}, - "pluginVersion": "7.1.1", - "targets": [ - { - "expr": "kube_hpa_status_current_replicas{hpa=~\".*\"} == kube_hpa_spec_min_replicas{hpa=~\".*\"}", - "instant": false, - "interval": "", - "legendFormat": "{{hpa}}", - "refId": "B" - } - ], - "title": "current==min", - "type": "timeseries" - } - ], - "title": "HPA", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 45, - "panels": [ - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 0, - "y": 26 - }, - "id": 47, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_job_status_succeeded{namespace=~\"$namespace\",cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Jobs Succeeded", - "type": "stat" - }, - { - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 26 - }, - "id": 62, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "(time()-kube_job_status_completion_time{namespace=~\"$namespace\",cluster=~\"$cluster\"})/60/60/24>1", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{job_name}}", - "refId": "A" - } - ], - "title": "Job last success time from now", - "transparent": true, - "type": "table" - }, - { - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 26 - }, - "id": 76, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "kube_job_status_failed{namespace=~\"$namespace\",cluster=~\"$cluster\"}>1", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{job_name}}", - "refId": "A" - } - ], - "title": "Job failed", - "transparent": true, - "type": "table" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 0, - "y": 37 - }, - "id": 48, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_job_status_active{namespace=~\"$namespace\",cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Jobs Active", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 0, - "y": 40 - }, - "id": 49, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_job_status_failed{namespace=~\"$namespace\",cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Jobs Failed", - "type": "stat" - } - ], - "title": "Jobs", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 64, - "panels": [ - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 27 - }, - "id": 66, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_persistentvolumeclaim_status_phase{phase=\"Bound\"}==1)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{persistentvolumeclaim=}}", - "refId": "A" - } - ], - "title": "pvc is bound", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 27 - }, - "id": 69, - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "sum(kube_persistentvolumeclaim_status_phase{phase!=\"Bound\"}==1)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{persistentvolumeclaim=}}", - "refId": "A" - } - ], - "title": "pvc is not bound", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 27 - }, - "id": 67, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "expr": "kube_persistentvolumeclaim_resource_requests_storage_bytes/1024/1024/1024", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{persistentvolumeclaim}}", - "refId": "A" - } - ], - "title": "pvc", - "type": "table" - } - ], - "title": "PVC", - "type": "row" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 41, - "tags": ["kubernetes", "kubernetes-app"], - "templating": { - "list": [ - { - "current": { - "text": "", - "value": "" - }, - "definition": "label_values(kube_node_info, cluster)", - "includeAll": false, - "name": "cluster", - "options": [], - "query": "label_values(kube_node_info, cluster)", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "label_values(kube_node_info{cluster=~\"$cluster\"}, node)", - "includeAll": true, - "name": "node", - "options": [], - "query": "label_values(kube_node_info{cluster=~\"$cluster\"}, node)", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "", - "includeAll": true, - "name": "namespace", - "options": [], - "query": "label_values(kube_namespace_created{cluster=~\"$cluster\"}, namespace)", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 2, - "name": "datasource", - "query": "Prometheus", - "skipUrlSync": true, - "type": "constant" - } - ] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "browser", - "title": "kube-state-metrics-v2", - "uid": "garysdevil-kube-state-metrics-v2", - "version": 1 - }, - "overwrite": true -} diff --git a/rootfs/usr/share/grafana/api/dashboards/main/drycc_component_health.json b/rootfs/usr/share/grafana/api/dashboards/main/drycc_component_health.json new file mode 100644 index 0000000..daa1104 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/main/drycc_component_health.json @@ -0,0 +1,2770 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 33, + "panels": [], + "targets": [ + { + "refId": "A" + } + ], + "title": "SUM", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 31, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_namespace", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",container!=\"\",namespace=\"drycc\"}[1m])) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["namespace"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total CPU Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 32, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_namespace", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",container!=\"\",namespace=\"drycc\"})", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["namespace"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 52, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 2 + }, + "hiddenSeries": false, + "id": 54, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",container=~\"^drycc-passport.*$\"}[1m])) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 2 + }, + "hiddenSeries": false, + "id": 56, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-passport.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "PASSPORT", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 34, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "hiddenSeries": false, + "id": 1, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-controller.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "hiddenSeries": false, + "id": 2, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-controller.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "CONTROLLER", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 35, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 4 + }, + "hiddenSeries": false, + "id": 3, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-builder.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 4 + }, + "hiddenSeries": false, + "id": 4, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-builder.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "BUILDER", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 36, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 6, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-database.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 12 + }, + "hiddenSeries": false, + "id": 5, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-database.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "DATABASE", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 37, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 13 + }, + "hiddenSeries": false, + "id": 19, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-logger-fluentbit.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 13 + }, + "hiddenSeries": false, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-logger-fluentbit.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "FLUENTBIT", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 38, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 25, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "datasource": null, + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-grafana.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 26, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "datasource": null, + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-grafana.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "GRAFANA", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 40, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 7, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=\"drycc-logger\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 8, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=\"drycc-logger\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "LOGGER", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 41, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 9, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 17 + }, + "hiddenSeries": false, + "id": 10, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "STORAGE", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 43, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 27, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-valkey.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 28, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-valkey.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "VALKEY", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 44, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 11, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-registry.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 22 + }, + "hiddenSeries": false, + "id": 12, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=\"drycc-registry\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "REGISTRY", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 45, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 16, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-registry-proxy.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 23 + }, + "hiddenSeries": false, + "id": 15, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-registry-proxy.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "REGISTRY-PROXY", + "type": "row" + } + ], + "refresh": "5s", + "schemaVersion": 37, + "style": "dark", + "tags": ["drycc"], + "templating": { + "list": [] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "browser", + "title": "Drycc Component Health", + "uid": "2ySBrTTMz", + "version": 1, + "weekStart": "" +} diff --git a/rootfs/usr/share/grafana/api/dashboards/main/drycc_database.json b/rootfs/usr/share/grafana/api/dashboards/main/drycc_database.json new file mode 100644 index 0000000..0ecb189 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/main/drycc_database.json @@ -0,0 +1,1327 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A quickstart to setup the Prometheus PostgreSQL Exporter with preconfigured dashboards, alerting rules, and recording rules.", + "editable": true, + "gnetId": 14114, + "graphTooltip": 0, + "id": 1, + "iteration": 1603191461722, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 20, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 1, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_fetched{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "fetched", + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["tup_fetched"], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": ["10s"], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_returned{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "returned", + "measurement": "postgresql", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["tup_fetched"], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": ["10s"], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_inserted{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "inserted", + "measurement": "postgresql", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["tup_fetched"], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": ["10s"], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_updated{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "updated", + "measurement": "postgresql", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["tup_fetched"], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": ["10s"], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_deleted{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "deleted", + "measurement": "postgresql", + "policy": "default", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["tup_fetched"], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": ["10s"], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rows", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 0, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "height": "55px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_xact_commit{datname=~\"$db\",instance=~\"$instance\"}[5m])) + sum(irate(pg_stat_database_xact_rollback{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "intervalFactor": 2, + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["xact_commit"], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": ["10s"], + "type": "non_negative_derivative" + } + ] + ], + "step": 1800, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + } + ], + "thresholds": "", + "title": "QPS", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "decimals": 1, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 2, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance=~'$instance'}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_alloc", + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["buffers_alloc"], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{instance=~'$instance'}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_backend_fsync", + "measurement": "postgresql", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["buffers_alloc"], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_backend{instance=~'$instance'}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_backend", + "measurement": "postgresql", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["buffers_alloc"], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_clean{instance=~'$instance'}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_clean", + "measurement": "postgresql", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["buffers_alloc"], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{instance=~'$instance'}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_checkpoint", + "measurement": "postgresql", + "policy": "default", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["buffers_alloc"], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Buffers", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 3, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "conflicts", + "dsType": "prometheus", + "expr": "sum(rate(pg_stat_database_deadlocks{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "deadlocks", + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["conflicts"], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "deadlocks", + "dsType": "prometheus", + "expr": "sum(rate(pg_stat_database_conflicts{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "conflicts", + "measurement": "postgresql", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["deadlocks"], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Conflicts/Deadlocks", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 12, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": true, + "pluginVersion": "7.2.1", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=~\"$instance\"}) / (sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=~\"$instance\"}) + sum(pg_stat_database_blks_read{datname=~\"$db\",instance=~\"$instance\"}))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "cache hit rate", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache hit ratio", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 13, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_stat_database_numbackends{datname=~\"$db\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{__name__}}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of active connections", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": ["postgres"], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(up{job=~\"postgres.*\"},instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "definition": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", + "hide": 0, + "includeAll": true, + "label": "db", + "multi": false, + "name": "db", + "options": [], + "query": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "Postgres Overview", + "value": "Postgres Overview" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "postgres", + "value": "postgres" + }, + "definition": "label_values(pg_up, job)", + "hide": 0, + "includeAll": false, + "label": "job", + "multi": false, + "name": "job", + "options": [ + { + "selected": true, + "text": "postgres", + "value": "postgres" + } + ], + "query": "label_values(pg_up, job)", + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "browser", + "title": "PostgreSQL Exporter Quickstart and Dashboard", + "uid": "wGgaPlciz", + "version": 5 +} diff --git a/rootfs/usr/share/grafana/api/dashboards/main/drycc_registry.json b/rootfs/usr/share/grafana/api/dashboards/main/drycc_registry.json new file mode 100644 index 0000000..fd6a364 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/main/drycc_registry.json @@ -0,0 +1,1419 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": 9621, + "graphTooltip": 0, + "id": null, + "iteration": 1547215355938, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 22, + "panels": [], + "title": "General Counters, CPU, Memory and File Descriptor Stats", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], + "description": "start time of the process", + "format": "dateTimeFromNow", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "110%", + "prefix": "", + "prefixFontSize": "110%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "process_start_time_seconds{app=\"$app\", instance=\"$instance\"} * 1000", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Start Time", + "type": "singlestat", + "valueFontSize": "110%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": ["#967302", "#967302", "#967302"], + "description": "The number of cache request received", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 4, + "y": 1 + }, + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "150%", + "prefix": "", + "prefixFontSize": "150%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "#eab839", + "full": false, + "lineColor": "#e5ac0e", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "registry_storage_cache_total{instance=\"$instance\",type=\"Request\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ type }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Total Cache Requests", + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": ["#1f78c1", "#1f78c1", "#1f78c1"], + "description": "The number of cache request received (hits)", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 9, + "y": 1 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "150%", + "prefix": "", + "prefixFontSize": "150%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "#5195ce", + "full": false, + "lineColor": "#0a50a1", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "registry_storage_cache_total{instance=\"$instance\",type=\"Hit\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ type }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Total Hit Cache Requests", + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": ["#890f02", "#890f02", "#890f02"], + "description": "The number of cache request received (miss)", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 14, + "y": 1 + }, + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "150%", + "prefix": "", + "prefixFontSize": "150%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "#e24d42", + "full": false, + "lineColor": "#e24d42", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "registry_storage_cache_total{instance=\"$instance\",type=\"Miss\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ type }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Total Missed Cache Requests", + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], + "description": "Process Resident Memory Usage", + "format": "decbytes", + "gauge": { + "maxValue": 1000, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 19, + "y": 1 + }, + "id": 24, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "150%", + "prefix": "", + "prefixFontSize": "150%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(process_resident_memory_bytes{instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Resident Memory Usage", + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Average user and system CPU time spent in seconds.", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(process_cpu_seconds_total{instance=\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "CPU Time", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Virtual and Resident memory size in bytes, averages over 5 min interval", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 4 + }, + "id": 32, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(process_resident_memory_bytes{instance=\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Resident Mem", + "refId": "A" + }, + { + "expr": "avg(rate(process_virtual_memory_bytes{instance=\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Virtual Mem", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Number of open file descriptors", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_open_fds{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Open FD", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Open File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 20, + "panels": [], + "title": "Registry Metrics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "The HTTP requests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 26, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 3, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(rate(registry_http_requests_total{instance=\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ handler }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "HTTP Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "The in-flight HTTP requests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 12 + }, + "id": 18, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "registry_http_in_flight_requests{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ handler }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "In-Flight HTTP Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "The rate of registry cache requests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 28, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 3, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(rate(registry_storage_cache_total{instance=\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Registry Cache Requests Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "increase(registry_storage_action_seconds_sum{instance=\"$instance\"}[2m]) * 1000", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{ action }}", + "refId": "A", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Registry Action Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 20 + }, + "heatmap": {}, + "highlightCards": true, + "id": 36, + "legend": { + "show": false + }, + "links": [], + "targets": [ + { + "expr": "rate(registry_http_request_duration_seconds_bucket{handler=\"blob\"}[10m])", + "format": "heatmap", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "HTTP Request Latencies in seconds (blob)", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": "0", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 20 + }, + "heatmap": {}, + "highlightCards": true, + "id": 38, + "legend": { + "show": false + }, + "links": [], + "targets": [ + { + "expr": "rate(registry_http_request_duration_seconds_bucket{handler=\"base\"}[10m])", + "format": "heatmap", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Upload HTTP Request Latencies in seconds (base)", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": "0", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 27 + }, + "heatmap": {}, + "highlightCards": true, + "id": 2, + "legend": { + "show": false + }, + "links": [], + "targets": [ + { + "expr": "rate(registry_http_request_duration_seconds_bucket{handler=\"blob_upload\"}[10m])", + "format": "heatmap", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Upload HTTP Request Latencies in seconds (blob_upload)", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": "0", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 27 + }, + "heatmap": {}, + "highlightCards": true, + "id": 42, + "legend": { + "show": false + }, + "links": [], + "targets": [ + { + "expr": "rate(registry_http_request_duration_seconds_bucket{handler=\"manifest\"}[10m])", + "format": "heatmap", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Manifest HTTP Request Latencies in seconds (manifest)", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": "0", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 27 + }, + "heatmap": {}, + "highlightCards": true, + "id": 40, + "legend": { + "show": false + }, + "links": [], + "targets": [ + { + "expr": "rate(registry_http_request_duration_seconds_bucket{handler=\"catalog\"}[10m])", + "format": "heatmap", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Catalog HTTP Request Latencies in seconds (catalog)", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": "0", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + } + ], + "refresh": "10s", + "schemaVersion": 16, + "style": "dark", + "tags": ["docker", "registry", "internals"], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [], + "query": "query_result(registry_http_in_flight_requests)", + "refresh": 2, + "regex": "/.*kubernetes_namespace=\"([^\"]+).*/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "app", + "options": [], + "query": "query_result(registry_http_in_flight_requests{kubernetes_namespace=\"$namespace\"})", + "refresh": 2, + "regex": "/.*app=\"([^\"]+)/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "query_result(up{app=\"$app\"})", + "refresh": 1, + "regex": "/.*instance=\"([^\"]+).*/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "", + "title": "Docker Registry", + "uid": "CoBSgj8iz", + "version": 19 +} diff --git a/rootfs/usr/share/grafana/api/dashboards/main/drycc_storage.json b/rootfs/usr/share/grafana/api/dashboards/main/drycc_storage.json new file mode 100644 index 0000000..abb55b9 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/main/drycc_storage.json @@ -0,0 +1,3258 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "MinIO Grafana Dashboard - https://min.io/", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 13502, + "graphTooltip": 0, + "id": null, + "links": [ + { + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": ["minio"], + "type": "dashboards" + } + ], + "liveNow": false, + "panels": [ + { + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dtdurations", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 1, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "time() - max(minio_node_process_starttime_seconds{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 65, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "sum by (instance) (minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total S3 Ingress", + "type": "stat" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "unit": "bytes", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 6, + "y": 0 + }, + "id": 50, + "interval": "1m", + "links": [], + "maxDataPoints": 100, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": ["percent"] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "exemplar": true, + "expr": "topk(1, sum(minio_cluster_capacity_usable_total_bytes{job=~\"$scrape_jobs\"}) by (instance)) - topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=~\"$scrape_jobs\"}) by (instance))", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "Used", + "refId": "A", + "step": 300 + }, + { + "exemplar": true, + "expr": "topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=~\"$scrape_jobs\"}) by (instance)) ", + "hide": false, + "interval": "1m", + "legendFormat": "Free", + "refId": "B" + } + ], + "title": "Capacity", + "type": "piechart" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Objects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": ["Usage"], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 10, + "y": 0 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "max(minio_cluster_usage_total_bytes{job=~\"$scrape_jobs\"})", + "interval": "", + "legendFormat": "Usage", + "range": true, + "refId": "A" + } + ], + "title": "Data Usage Growth", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 16, + "y": 0 + }, + "id": 52, + "links": [], + "options": { + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_objects_size_distribution{job=~\"$scrape_jobs\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{range}}", + "refId": "A", + "step": 300, + "useBackend": false + } + ], + "title": "Object Size Distribution", + "type": "bargauge" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 61, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{server}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Open FDs ", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 3 + }, + "id": 64, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "sum by (instance) (minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total S3 Egress", + "type": "stat" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 3 + }, + "id": 62, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "minio_node_go_routine_total{job=~\"$scrape_jobs\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{server}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Goroutines", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bool_on_off", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 6 + }, + "id": 94, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_health_status{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "Pool: {{pool}} Set: {{set}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Cluster Health Status", + "transformations": [], + "type": "stat" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 6 + }, + "id": 78, + "links": [], + "maxDataPoints": 100, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "max(minio_cluster_drive_online_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".", + "metric": "process_start_time_seconds", + "range": false, + "refId": "A", + "step": 60 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "max(minio_cluster_drive_offline_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": ".", + "range": false, + "refId": "B" + } + ], + "title": "Total Online/Offline Drives", + "type": "gauge" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "dark-yellow", + "value": 75000000 + }, + { + "color": "dark-red", + "value": 100000000 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 6 + }, + "id": 66, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "max(minio_cluster_bucket_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of Buckets", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 9, + "y": 6 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "sum by (server) (rate(minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Received [{{server}}]", + "refId": "A" + } + ], + "title": "S3 API Ingress Rate ", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "sum by (server) (rate(minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Sent [{{server}}]", + "refId": "A" + } + ], + "title": "S3 API Egress Rate ", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 8 + }, + "id": 53, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "max(minio_cluster_nodes_online_total{job=~\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total Online Servers", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "dark-yellow", + "value": 75000000 + }, + { + "color": "dark-red", + "value": 100000000 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 9 + }, + "id": 44, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "max(minio_cluster_usage_object_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of Objects", + "type": "stat" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ns", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 10 + }, + "id": 80, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "max(minio_heal_time_last_activity_nano_seconds{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Time Since Last Heal", + "type": "stat" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ns", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 3, + "y": 10 + }, + "id": 81, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "max(minio_usage_last_activity_nano_seconds{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Time Since Last Scan", + "type": "stat" + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 9, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 API Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 7, + "x": 9, + "y": 12 + }, + "hiddenSeries": false, + "id": 88, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_4xx_errors_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 API Request Error Rate (4xx)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 86, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_5xx_errors_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 API Request Error Rate (5xx)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 99, + "options": { + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_online_drives{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Online Drives", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_read_quorum{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Read Quorum", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_write_quorum{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Write Quorum", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_healing_drives{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Healing Drives", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_status{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Status", + "range": true, + "refId": "E", + "useBackend": false + } + ], + "title": "Health Breakdown", + "type": "bargauge" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 76, + "options": { + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "minio_node_process_resident_memory_bytes{job=~\"$scrape_jobs\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{server}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory Usage ", + "type": "bargauge" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 73, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(minio_node_io_rchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Node RChar [{{server}}]", + "refId": "A" + }, + { + "exemplar": true, + "expr": "rate(minio_node_io_wchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Node WChar [{{server}}]", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Read, Write I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:381", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:382", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 77, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "rate(minio_node_process_cpu_total_seconds{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{server}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "description": "Total number of bytes received and sent on MinIO cluster", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 17, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "rate(minio_inter_node_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Internode Bytes Received [{{server}}]", + "metric": "minio_http_requests_duration_seconds_count", + "range": true, + "refId": "A", + "step": 4 + }, + { + "exemplar": true, + "expr": "rate(minio_inter_node_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Internode Bytes Sent [{{server}}]", + "refId": "B" + } + ], + "title": "Internode Traffic", + "type": "timeseries" + }, + { + "aliasColors": { + "available 10.13.1.25:9000": "green", + "used 10.13.1.25:9000": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "", + "fieldConfig": { + "defaults": { + "links": [], + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 33 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", + "interval": "", + "legendFormat": "Open FDs [{{server}}]", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Offline 10.13.1.25:9000": "dark-red", + "Total 10.13.1.25:9000": "blue" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "description": "Number of online drives per MinIO Server", + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 40 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(minio_node_syscall_read_total{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Read Syscalls [{{server}}]", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + }, + { + "exemplar": true, + "expr": "rate(minio_node_syscall_write_total{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Write Syscalls [{{server}}]", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Syscalls", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:185", + "decimals": 0, + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:186", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 40 + }, + "hiddenSeries": false, + "id": 95, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(minio_node_scanner_objects_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Scanned Objects", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 75, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(minio_node_scanner_versions_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Scanned Versions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 47 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(minio_node_scanner_directories_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Scanned Directories", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "dtdurations", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 54 + }, + "id": 89, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "exemplar": true, + "expr": "minio_cluster_kms_uptime{job=~\"$scrape_jobs\"}", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "minio_cluster_kms_uptime", + "refId": "A", + "step": 60 + } + ], + "title": "KMS Uptime", + "type": "stat" + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 54 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (server) (increase(minio_cluster_kms_request_error{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "KMS Request 4xx Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unit": "bool_on_off", + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 54 + }, + "hiddenSeries": false, + "id": 90, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (server) (minio_cluster_kms_online{job=~\"$scrape_jobs\"})", + "interval": "1m", + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "KMS Online(1)/Offline(0)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "bool_on_off", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 54 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(minio_node_scanner_bucket_scans_finished{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Bucket Scans Finished", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 58 + }, + "hiddenSeries": false, + "id": 92, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (server) (increase(minio_cluster_kms_request_failure{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "KMS Request 5xx Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 58 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (server) (rate(minio_cluster_kms_request_success{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "KMS Request Success [{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "KMS Request Success Rate ", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 61 + }, + "hiddenSeries": false, + "id": 97, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(minio_node_scanner_bucket_scans_started{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Bucket Scans Started", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": ["minio"], + "templating": { + "list": [ + { + "current": {}, + "definition": "label_values(job)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "scrape_jobs", + "options": [], + "query": { + "query": "label_values(job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "", + "title": "MinIO Dashboard", + "uid": "TgmJnqnnk", + "version": 54, + "weekStart": "" +} diff --git a/rootfs/usr/share/grafana/api/dashboards/main/drycc_valkey.json b/rootfs/usr/share/grafana/api/dashboards/main/drycc_valkey.json new file mode 100644 index 0000000..043342c --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/main/drycc_valkey.json @@ -0,0 +1,1364 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Redis Dashboard for Prometheus Redis Exporter 1.x", + "editable": true, + "gnetId": 763, + "graphTooltip": 1, + "id": null, + "iteration": 1602758020790, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "Performance", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average taken across instances", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 18, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(redis_commands_total{instance=~\"$instance\"} [$__rate_interval])) by (cmd)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{cmd}}", + "metric": "redis_command_calls_total", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Commands per second", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average taken across instances", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 1 + }, + "hiddenSeries": false, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(redis_commands_duration_seconds_total{instance=~\"$instance\"}[$__rate_interval])) by (cmd)\n /\navg(irate(redis_commands_total{instance=~\"$instance\"}[$__rate_interval])) by (cmd)\n", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ cmd }}", + "metric": "redis_command_calls_total", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Command latency per second", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Hit ratio": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "description": "Hit rate shows the percentage of key space lookups that hit a key.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 1 + }, + "hiddenSeries": false, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Target/", + "color": "#56A64B", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "linewidth": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(redis_keyspace_hits_total{instance=~\"$instance\"}[$__rate_interval]) / (irate(redis_keyspace_misses_total{instance=~\"$instance\"}[$__rate_interval]) + irate(redis_keyspace_hits_total{instance=~\"$instance\"}[$__rate_interval]))) by (instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "", + "refId": "A", + "step": 240, + "target": "" + }, + { + "expr": "1", + "interval": "", + "legendFormat": "Target hit ratio for cache", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Hit ratio per instance", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 22, + "panels": [], + "title": "Memory", + "type": "row" + }, + { + "aliasColors": { + "max": "#BF1B00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total taken across instances", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 7, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/max/", + "color": "#E02F44", + "dashes": true, + "fill": 0, + "linewidth": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(redis_memory_used_bytes{instance=~\"$instance\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Used Memory", + "metric": "", + "refId": "A", + "step": 240, + "target": "" + }, + { + "expr": "sum(redis_memory_max_bytes{instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Configured max memory", + "refId": "B", + "step": 240 + }, + { + "expr": "sum(redis_memory_used_rss_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Used RSS memory", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Recommend restart redis": "red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 9 + }, + "hiddenSeries": false, + "id": 10, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/restart/", + "color": "#E02F44", + "dashes": true, + "fill": 0, + "linewidth": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "redis_memory_fragmentation_ratio{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory fragmentation ratio per instance", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Evictions": "red", + "evicts": "#890F02", + "memcached_items_evicted_total{instance=\"172.17.0.1:9150\",job=\"prometheus\"}": "#890F02", + "reclaims": "#3F6833", + "{container=\"redis-exporter\", instance=\"redis-86cb5d76d7-fcdln:redis-exporter:redis-metrics\", job=\"default/redis\", namespace=\"default\", pod=\"redis-86cb5d76d7-fcdln\"}": "red", + "{instance=\"redis-86cb5d76d7-fcdln:redis-exporter:redis-metrics\"}": "red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 9 + }, + "hiddenSeries": false, + "id": 8, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "reclaims", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(redis_evicted_keys_total{instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Key evictions per second per instance", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 26, + "panels": [], + "title": "Basic activity", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Sum taken across instances", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(redis_connected_clients{instance=~\"$instance\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Connected", + "refId": "A" + }, + { + "expr": "sum(redis_blocked_clients{instance=~\"$instance\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Blocked", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Connected/Blocked Clients", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "db1": "yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Sum taken across instances", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 17 + }, + "hiddenSeries": false, + "id": 5, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum (redis_db_keys{instance=~\"$instance\"}) by (db)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ db }}", + "refId": "A", + "step": 240, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Items per DB", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Sum taken across instances", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 17 + }, + "hiddenSeries": false, + "id": 13, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum (redis_db_keys{instance=~\"$instance\"}) - sum (redis_db_keys_expiring{instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Not expiring", + "refId": "A", + "step": 240, + "target": "" + }, + { + "expr": "sum(redis_db_keys_expiring{instance=~\"$instance\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Expiring", + "metric": "", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Expiring vs Not-Expiring Keys", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "This metric will only be non-zero if the instance is a master", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(redis_connected_slaves{instance=~\"$instance\"}) by (instance)", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Connected slaves by instance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "This metric is only exported if the instance is a slave.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 24 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "redis_master_last_io_seconds_ago{instance=~\"$instance\"}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Time since last master connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 25, + "style": "dark", + "tags": ["prometheus", "redis"], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "tags": [], + "text": "redisdb-7d6b98cd98-kjt5x + redisdb-7d6b98cd98-pbkg6", + "value": ["redisdb-7d6b98cd98-kjt5x", "redisdb-7d6b98cd98-pbkg6"] + }, + "datasource": "${datasource}", + "definition": "label_values(redis_up, instance)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(redis_up, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "browser", + "title": "Redis Dashboard for Prometheus Redis Exporter 1.x", + "uid": "bRd48yKMdd", + "version": 5 +} diff --git a/rootfs/usr/share/grafana/api/dashboards/main/drycc_victoriametrics_cluster.json b/rootfs/usr/share/grafana/api/dashboards/main/drycc_victoriametrics_cluster.json new file mode 100644 index 0000000..629c4b4 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/main/drycc_victoriametrics_cluster.json @@ -0,0 +1,9920 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "enable": true, + "expr": "sum(ALERTS{job=~\"$job\", instance=~\"$instance\", alertgroup=\"vmcluster\",alertstate=\"firing\",show_at=\"dashboard\"}) by(alertname)", + "hide": false, + "iconColor": "red", + "name": "alerts", + "titleFormat": "{{alertname}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "enable": true, + "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(version))", + "hide": true, + "iconColor": "dark-blue", + "name": "version change", + "textFormat": "{{version}}", + "titleFormat": "Version change" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "enable": true, + "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job)", + "hide": false, + "iconColor": "dark-yellow", + "name": "restarts", + "textFormat": "{{job}} restarted" + } + ] + }, + "description": "Overview for cluster VictoriaMetrics v1.102.0 or higher", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 1, + "links": [ + { + "icon": "doc", + "tags": [], + "targetBlank": true, + "title": "Cluster Wiki", + "type": "link", + "url": "https://docs.victoriametrics.com/cluster-victoriametrics" + }, + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "Found a bug?", + "type": "link", + "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues" + }, + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "New releases", + "type": "link", + "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 137, + "panels": [], + "title": "Stats", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "How many [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) are in storage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 131, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "sum(vm_rows{job=~\"$job_storage\", type!~\"indexdb.*\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total datapoints", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the datapoints ingestion rate, including replication factor.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 124, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(vm_vminsert_metrics_read_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) ", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Ingestion rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of HTTP read requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 130, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_http_requests_total{job=~\"$job\", instance=~\"$instance\", path=~\"/select/.*\"}[$__rate_interval]))", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Read requests", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Total number of available CPUs for all VM components. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 126, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Available CPU", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series) with new data points inserted during the last hour. High value may result in ingestion slowdown.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 34, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(max_over_time(vm_cache_entries{job=~\"$job\", instance=~\"$instance\", type=\"storage/hour_metric_ids\"}[1h]))", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Active series", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Total amount of used disk space", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 4 + }, + "id": 35, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": false, + "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Disk space usage", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Average disk usage per datapoint.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 4 + }, + "id": 112, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\"}) / sum(vm_rows{job=~\"$job_storage\", type!~\"indexdb.*\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Bytes per point", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Total size of available memory for all VM components.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 4 + }, + "id": 128, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Available memory", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "minWidth": 50 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Count" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 149, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Count" + } + ] + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 16, + "x": 8, + "y": 7 + }, + "id": 62, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(min_over_time(up{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)", + "format": "time_series", + "instant": false, + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "Uptime ($job)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "How many [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) are inserted into cluster per second by protocol before the replication. Check vminsert metrics if there are any issues with ingestion.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (type) > 0 ", + "interval": "", + "legendFormat": "{{type}}", + "range": true, + "refId": "A" + } + ], + "title": "Datapoints ingestion rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series) with new data points inserted during the last hour across all storage nodes. High value may result in ingestion slowdown and high memory usage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 12, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(vm_cache_entries{job=~\"$job\", instance=~\"$instance\", type=\"storage/hour_metric_ids\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active time series", + "refId": "A" + } + ], + "title": "Active time series ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The less time it takes is better.\n* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 8, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(vm_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", quantile=\"0.99\"}) by (path) > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Query duration 0.99 quantile ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 52, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, path) > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{path}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Requests error rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=203&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 104, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, level) > 0", + "format": "time_series", + "hide": false, + "interval": "5m", + "intervalFactor": 1, + "legendFormat": "{{job}} - {{level}}", + "range": true, + "refId": "A" + } + ], + "title": "Logging rate", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 46, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Percentage of used RSS memory (resident).\nThe RSS memory shows the amount of memory recently accessed by the application. It includes anonymous memory and data from recently accessed files (aka page cache).\nThe application's performance will significantly degrade when memory usage is close to 100%.\n\nClick on the line and choose Drilldown to show memory usage per instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 66, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RSS memory % usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%\n\nClick on the line and choose Drilldown to show memory usage per instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=190&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 138, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RSS anonymous memory % usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 160 + }, + "id": 64, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of bytes read/write from the storage layer.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/read .*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 160 + }, + "id": 122, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "read {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Disk writes/reads ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percentage of open file descriptors compared to the limit set in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/max.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 168 + }, + "id": 117, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(process_open_fds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_max_fds{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Open FDs usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of read/write syscalls such as read, pread, write, pwrite.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/read .*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 168 + }, + "id": 204, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(process_io_read_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "read {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_write_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Disk write/read calls ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 176 + }, + "id": 68, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(max_over_time(go_goroutines{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 176 + }, + "id": 119, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(max_over_time(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP connections ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 184 + }, + "id": 70, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(max_over_time(process_num_threads{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Threads ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 184 + }, + "id": 120, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "TCP connections rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/troubleshooting/ ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 192 + }, + "id": 210, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(\n rate(go_gc_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n ) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU spent on GC ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 192 + }, + "id": 213, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, instance, le))) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Go scheduling latency", + "type": "timeseries" + } + ], + "title": "Resource usage ($job)", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 106, + "panels": [ + { + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 211, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "See [Troubleshooting](https://docs.victoriametrics.com/troubleshooting/) docs.", + "mode": "markdown" + }, + "pluginVersion": "11.5.0", + "title": "", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate and total number of new series created over last 24h.\n\nHigh [churn rate](https://docs.victoriametrics.com/faq/#what-is-high-churn-rate) tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected [cardinality](https://docs.victoriametrics.com/keyconcepts/#cardinality) \"explosions\".\n\nThe higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.\n\nTo investigate stats about most expensive series use `api/v1/status/tsdb` handler. More details here https://docs.victoriametrics.com/cluster-victoriametrics/#url-format\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://valyala.medium.com/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "new series over 24h" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisSoftMin", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 104 + }, + "id": 102, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "sum(rate(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "churn rate", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "sum(increase(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[24h]))", + "hide": false, + "interval": "", + "legendFormat": "new series over 24h", + "refId": "B" + } + ], + "title": "Churn rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The percentage of [slow inserts](https://docs.victoriametrics.com/faq/#what-is-a-slow-insert) compared to the total ingestion rate. \\n\\nThe lower the better. \\n\\nIn short, slow insert is a cache miss. There are following reasons for slow inserts to go up: \\n* Ingestion of completely new, not seen before time series;\\n* [Re-routing](https://docs.victoriametrics.com/cluster-victoriametrics/#cluster-availability) of series when one or more vmstorage nodes are unavailable;\\n* Not enough memory to maintain big enough caches for the current workload.\\n\\nIf percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series). \\n\\nSee [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 104 + }, + "id": 108, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(\n rate(vm_slow_row_inserts_total{job=~\"$job_storage\"}[$__rate_interval]) \n / rate(vm_rows_added_to_storage_total{job=~\"$job_storage\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "slow inserts", + "range": true, + "refId": "A" + } + ], + "title": "Slow inserts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Merge assist happens when vmstorage can't keep up with merging parts. This is usually a sign of overload for vmstorage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 112 + }, + "id": 170, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(vm_assisted_merges_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type) > 0", + "format": "time_series", + "interval": "5m", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Assisted merges ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows % of slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.\n\nThe less value is better.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 112 + }, + "id": 107, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_slow_queries_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval]))\n/\nsum(rate(vm_http_requests_total{job=~\"$job_select\", instance=~\"$instance\", path=~\"/select/.*\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "slow queries %", + "range": true, + "refId": "A" + } + ], + "title": "Slow queries % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 120 + }, + "id": 144, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"} \n /\n vm_cache_size_max_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(type)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Cache usage % by type ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows cache miss ratio. Lower is better.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 120 + }, + "id": 58, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n rate(vm_cache_misses_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n rate(vm_cache_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n) by(type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Cache miss ratio ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The value is above 0 when the vmstorage at the given `addr` communicates to the given vminsert node that it cannot accept new data because it is in the read-only mode.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 129 + }, + "id": 142, + "links": [ + { + "targetBlank": true, + "title": "Readonly mode", + "url": "https://docs.victoriametrics.com/cluster-victoriametrics/#readonly-mode" + } + ], + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vm_rpc_vmstorage_is_read_only{job=~\"$job_insert\", instance=~\"$instance\"}) by(instance, addr) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} => {{addr}}", + "range": true, + "refId": "A" + } + ], + "title": "Storage in readonly status for vminsert ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows rate of deduplicated samples during [deduplication](https://docs.victoriametrics.com/#deduplication) or [downsampling](https://docs.victoriametrics.com/#downsampling), according to the configured `-dedup.minScrapeInterval` or `-downsampling.period`.\n\nShould be empty if `-dedup.minScrapeInterval` and `-downsampling.period` isn't set.\n\nDeduplication happens in two places:\n* `type=\"merge\"` - during [background merges](https://docs.victoriametrics.com/#storage) by vmstorages. Deduplication during merges is permanent.\n* `type=\"select\"` - during [read queries](https://docs.victoriametrics.com/keyconcepts/#query-data) by vmselects. Deduplication happens in-flight and has no permanent effect.\n\nThe more samples need to be deduplicated, the higher will be resource usage on vmselects or vmstorages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 129 + }, + "id": 215, + "links": [], + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_deduplicated_samples_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) >0", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Deduplication rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows how many samples were ignored or dropped on insertion due to various reasons:\n* timestamp out of retention period or timestamp in future;\n* invalid metric name;\n* exceeding limit for labels length or number;\n* dropped by [relabeling configuration](https://docs.victoriametrics.com/#relabeling).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 137 + }, + "id": 135, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(vm_rows_ignored_total{job=~\"$job\", instance=~\"$instance\"}[1h])) by (reason)", + "hide": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(increase(vm_relabel_metrics_dropped_total{job=~\"$job\", instance=~\"$instance\"}[1h]))", + "hide": false, + "instant": false, + "legendFormat": "relabeling", + "range": true, + "refId": "B" + } + ], + "title": "Samples dropped for last 1h ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of partial results served by `vmselects`.\n\n`vmselect` continues serving queries if at least a single vmstorage nodes is available. It marks responses as `partial` for queries served from the remaining healthy vmstorage nodes.\n\nIf you prefer consistency over availability then run vmselect nodes with `-search.denyPartialResponse` cmd-line flag. In this case vmselect returns an error if at least a single vmstorage node is unavailable.\n\nSee more at [cluster availability](https://docs.victoriametrics.com/cluster-victoriametrics/#cluster-availability).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 137 + }, + "id": 217, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(increase(vm_partial_results_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0", + "format": "time_series", + "instant": false, + "legendFormat": "partial results", + "refId": "A" + } + ], + "title": "Partial query results ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 145 + }, + "id": 183, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "job" + } + ] + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(flag{is_set=\"true\", job=~\"$job\", instance=~\"$instance\"}) by(job, instance, name, value)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Non-default flags", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "instance": { + "aggregations": ["uniqueValues"], + "operation": "aggregate" + }, + "job": { + "aggregations": [], + "operation": "groupby" + }, + "name": { + "aggregations": [], + "operation": "groupby" + }, + "value": { + "aggregations": [], + "operation": "groupby" + } + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 145 + }, + "id": 214, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "instant": false, + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "Restarts ($job)", + "type": "timeseries" + } + ], + "title": "Troubleshooting", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 48, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows number of pushed and sent rows. \n* `Pushed rows` - rows added to internal inserter buffers before send\n* `Sent rows` - successfully transmitted rows to storage nodes\n\nPlease note, it could be that `Sent > Pushed` because of the replication factor.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1331 + }, + "id": 76, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_rpc_rows_pushed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pushed", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_rpc_rows_sent_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Sent", + "range": true, + "refId": "E" + } + ], + "title": "Rows ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Rate of RPC errors by type:\n* `Connection` - the number of connection errors to vmstorage node\n* `Dial` - the number of dial errors to vmstorage node.\n* `Handshake` - the number of handshake errors to vmstorage node\n* `Rerouted` - errors appeared during rerouting of rows from un-healthy storage node to a healthy one.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1331 + }, + "id": 86, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_rpc_connection_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Connection", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(rate(vm_rpc_dial_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Dial", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(rate(vm_rpc_handshake_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Handshake", + "refId": "E" + } + ], + "title": "RPC errors ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The number of rows rerouted to the vmstorage node from other nodes when they were unhealthy.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1340 + }, + "id": 80, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_rpc_rows_rerouted_to_here_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Rows ($instance) rerouted to ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The number of rows rerouted from the vmstorage node to healthy nodes when the given node was unhealthy.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1340 + }, + "id": 78, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "sum(rate(vm_rpc_rows_rerouted_from_here_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{addr}}", + "refId": "A" + } + ], + "title": "Rows ($instance) rerouted from", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The number of rows or bytes that vminsert internal buffer contains at the moment.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "bytes" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1348 + }, + "id": 82, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(vm_rpc_buf_pending_bytes{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "bytes", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(vm_rpc_rows_pending{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "rows", + "refId": "B" + } + ], + "title": "Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Network usage by internal VictoriaMetrics RPC protocol", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1348 + }, + "id": 74, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) * 8", + "legendFormat": "network usage", + "range": true, + "refId": "A" + } + ], + "title": "RPC network usage ($instance)", + "type": "timeseries" + } + ], + "title": "Interconnection ($job)", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 24, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of datapoints ingested into storage nodes per second. This metric doesn't show all stored datapoints since some of them may be dropped because of wrong timestamps or decode errors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 100, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_vminsert_metrics_read_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) ", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "ingestion rate", + "range": true, + "refId": "A" + } + ], + "title": "Ingestion rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the approx time needed to reach 100% of disk capacity for at least one vmstorage node based on the following params:\n* free disk space (after -storage.minFreeDiskSpaceBytes);\n* row ingestion rate;\n* compression.\n\nNote: this panel doesn't account for deduplication process.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=196&${__url_time_range}&${__all_variables}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 113, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "min"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "min((vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"} - vm_free_disk_space_limit_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) \n/ \nignoring(path) (\n rate(vm_rows_added_to_storage_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d])\n * scalar(\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n / \n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n )\n))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "min ETA", + "range": true, + "refId": "A" + } + ], + "title": "Storage full ETA ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=$job_storage&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 151, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"}\n)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "min(\n rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "quantile(0.5,\n rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "median", + "range": true, + "refId": "C" + } + ], + "title": "CPU usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Percentage of used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=$job_storage&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 167, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}\n)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(\n max_over_time(process_resident_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "quantile(0.5,\n max_over_time(process_resident_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "median", + "range": true, + "refId": "C" + } + ], + "title": "Memory usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. The `max` is an internal limit and can't be changed. It is always equal to the number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 212, + "links": [ + { + "targetBlank": true, + "title": "Related discussion", + "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/632" + } + ], + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(vm_concurrent_insert_current{job=~\"$job_storage\", \n instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "current", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(vm_concurrent_insert_capacity{job=~\"$job_storage\", instance=~\"$instance\"})", + "hide": false, + "interval": "", + "legendFormat": "max", + "range": true, + "refId": "B" + } + ], + "title": "Concurrent flushes on disk ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of concurrently executed read requests, where:\n* `max` - equal to `-search.maxConcurrentRequest` cmd-line flag;\n* `current` - current number of concurrent select requests executed by storage.\n\nEach concurrent select request requires RAM, CPU and disk IO resources during processing. Too many concurrent requests could result into OOM exceptions.\n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row).\n\nSee more at https://docs.victoriametrics.com/cluster-victoriametrics/#resource-usage-limits", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 133, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(vm_vmselect_concurrent_requests_current{job=~\"$job_storage\", \n instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "current", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(vm_vmselect_concurrent_requests_capacity{job=~\"$job_storage\", instance=~\"$instance\"})", + "hide": false, + "interval": "", + "legendFormat": "max", + "range": true, + "refId": "B" + } + ], + "title": "Concurrent selects ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The max number of on-going merges across storage nodes.\n The drastic change in number of merges could be a sign of on-going deduplication/downsampling activity.\n It is expected to have high numbers for `storage/small` metric.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 54, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(max_over_time(vm_active_merges{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active merges ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The number of rows merged per second by storage nodes. Merge speed depends on available CPU and disk IO bandwidth.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 55, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_rows_merged_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Merge speed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=200&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 20, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "min(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "quantile(0.5,\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "median", + "range": true, + "refId": "C" + } + ], + "title": "Disk space usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The max number of data parts of LSM tree across all storage nodes in the last [partition](https://docs.victoriametrics.com/#storage).\nIncrease in number of parts (the hard limit is 512) is an evidence of slow merge performance - check the resource utilization.\n* `indexdb` - inverted index\n* `storage/small` - recently added parts of data ingested into storage (hot data)\n* `storage/big` - small parts gradually merged into bigger parts (cold data)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 22, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(vm_parts{job=~\"$job_storage\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(type)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(vm_last_partition_parts{job=~\"$job_storage\", instance=~\"$instance\"}) by(type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "LSM parts max by type ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percentage of used disk space by type: datapoints or indexdb. Normally, indexdb takes much less space comparing to datapoints. But with high [churn rate](https://docs.victoriametrics.com/faq/#what-is-high-churn-rate) the size of the indexdb could grow significantly.\n\nThe sum of the % can be > 100% since panel shows max % per-job and per-instance. It means different instance can have different ratio between datapoints and indexdb size.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=201&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 202, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(job, instance)\n / \n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "indexdb", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"}) by(job, instance)\n / \n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "datapoints", + "range": true, + "refId": "B" + } + ], + "title": "Disk space usage % by type ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `3*`, since VictoriaMetrics pushes pending data to persistent storage every two seconds. The index datapoints value in general is much lower.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "pending index entries" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "decimals", + "value": 3 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 14, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(max_over_time(vm_pending_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"storage\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "pending datapoints", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(max_over_time(vm_pending_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "pending index entries", + "range": true, + "refId": "B" + } + ], + "title": "Pending datapoints ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows network usage by vmstorage services.\n* Writes show traffic sent to clients.\n* Reads show traffic received from clients.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 88 + }, + "id": 206, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(name) * 8 > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "read from {{name}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(name) * 8 > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "write to {{name}}", + "range": true, + "refId": "B" + } + ], + "title": "Network usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Value > 0 means vmstorage is in readonly mode.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 88 + }, + "id": 141, + "links": [ + { + "targetBlank": true, + "title": "Readonly mode", + "url": "https://docs.victoriametrics.com/cluster-victoriametrics/#readonly-mode" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "vm_storage_is_read_only{job=~\"$job_storage\", instance=~\"$instance\"} > 0", + "interval": "", + "legendFormat": "{{ instance }}", + "range": true, + "refId": "A" + } + ], + "title": "Readonly mode", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the max number of existing [snapshots](https://docs.victoriametrics.com/#how-to-work-with-snapshots) across vmstorages.\n\nRecently made snapshots do not occupy disk space. But with time, as snapshots become older, they start to occupy more and more disk space.\n\nIt is recommended deleting old snapshots when they are no longer needed to free up disk space.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 96 + }, + "id": 216, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(vm_snapshots{job=~\"$job_storage\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "snapshots", + "range": true, + "refId": "A" + } + ], + "title": "Number of snapshots", + "type": "timeseries" + } + ], + "title": "vmstorage ($instance)", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 42, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Request rate accepted by vmselect nodes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1344 + }, + "id": 92, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_http_requests_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Requests rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the max number of concurrent selects across instances.\n* `max` limit can be configured via `search.maxConcurrentRequests` flag\n* `current` shows the current number of goroutines busy with processing requests\n\nWhen `current` hits `max` constantly, it means one or more vmselect nodes are overloaded with number of requests. If you observe that CPU for vmselects is saturated, consider adding more vmselect replicas or increase CPU resources. If CPU panel shows a plenty of free resources - try increasing `search.maxConcurrentRequests`.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1344 + }, + "id": 95, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(max_over_time(vm_concurrent_select_current{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) ", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "current", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "min(vm_concurrent_select_capacity{job=~\"$job_select\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "B" + } + ], + "title": "Concurrent selects ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=$job_select&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1352 + }, + "id": 163, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_select\", instance=~\"$instance\"}\n)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(\n rate(process_cpu_seconds_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_select\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "quantile(0.5,\n rate(process_cpu_seconds_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_select\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "median", + "range": true, + "refId": "C" + } + ], + "title": "CPU usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=$job_select&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1352 + }, + "id": 165, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}\n)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(\n max_over_time(process_resident_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "quantile(0.5,\n max_over_time(process_resident_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "median", + "range": true, + "refId": "C" + } + ], + "title": "Memory usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "99th percentile of the number of time series read per query.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1360 + }, + "id": 178, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "series", + "range": true, + "refId": "A" + } + ], + "title": "Series read per query ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "99th percentile of number of [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) read per queried time series.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1360 + }, + "id": 180, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "datapoints", + "range": true, + "refId": "A" + } + ], + "title": "Datapoints read per series ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "99th percentile of number of [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) read per query.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1368 + }, + "id": 179, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "datapoints", + "range": true, + "refId": "A" + } + ], + "title": "Datapoints read per query ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "99th percentile of number of [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) scanner per query.\n\nThis number can exceed number of DatapointsReadPerQuery if `step` query arg passed to [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) is smaller than the lookbehind window set in square brackets of [rollup function](https://docs.victoriametrics.com/metricsql/#rollup-functions). For example, if `increase(some_metric[1h])` is executed with the `step=5m`, then the same [data samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on a hour time range are scanned `1h/5m=12` times. See [this article](https://valyala.medium.com/how-to-optimize-promql-and-metricsql-queries-85a1b75bf986) for details.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1368 + }, + "id": 181, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "datapoints", + "range": true, + "refId": "A" + } + ], + "title": "Datapoints scanned per query ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows network usage between vmselects and clients, such as vmalert, Grafana, vmui, etc.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1376 + }, + "id": 93, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "read from client", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "write to client", + "range": true, + "refId": "B" + } + ], + "title": "Network usage: clients ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows network usage between vmselects and vmstorages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1376 + }, + "id": 207, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcpdialer_read_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "read from vmstorage", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "write to vmstorage", + "range": true, + "refId": "B" + } + ], + "title": "Network usage: vmstorage ($instance)", + "type": "timeseries" + } + ], + "title": "vmselect ($instance)", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 40, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1345 + }, + "id": 97, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_http_requests_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Requests rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the max number of ongoing insertions.\n* `max` - equal to number of CPU * 2 by default. May be configured with `maxConcurrentInserts` flag;\n* `current` - current number of goroutines busy with processing requests.\n\n`-maxConcurrentInserts` limits the number of insert requests which may be actively processed at any given point in time. All the other insert requests are queued for up to `-insert.maxQueueDuration` in the hope they will get a chance to be processed. This queue is used mostly for absorbing spikes for incoming insert request rate.\n\nWhen `current` hits `max` constantly, it means vminsert node is overloaded and requires more CPU or higher limits.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1345 + }, + "id": 99, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(max_over_time(vm_concurrent_insert_current{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "current", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "min(vm_concurrent_insert_capacity{job=~\"$job_insert\", instance=~\"$instance\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "B" + } + ], + "title": "Concurrent inserts ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=$job_insert&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1353 + }, + "id": 185, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_insert\", instance=~\"$instance\"}\n)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(\n rate(process_cpu_seconds_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_insert\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "quantile(0.5,\n rate(process_cpu_seconds_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_insert\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "median", + "range": true, + "refId": "C" + } + ], + "title": "CPU usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=$job_insert&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1353 + }, + "id": 187, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}\n)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(\n max_over_time(process_resident_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "quantile(0.5,\n max_over_time(process_resident_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "median", + "range": true, + "refId": "C" + } + ], + "title": "Memory usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the saturation level of connection between vminsert and vmstorage components. \n\nIf the threshold of 0.9sec is reached, then the connection is saturated by more than 90% and vminsert won't be able to keep up. This usually means that either vminsert or vmstorage nodes are struggling with the load. Verify CPU/mem saturation of both components and network saturation between them.\nIf vminsert resources are saturated - consider adding more resources or scale vminserts horizontally.\n\nIf vminsert resources and network are fine, check vmstorage metrics for anomalies.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1361 + }, + "id": 139, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(rate(vm_rpc_send_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Storage connection saturation ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows when vmstorage node is unreachable for vminsert.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1361 + }, + "id": 114, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "vm_rpc_vmstorage_is_reachable{job=~\"$job\", instance=~\"$instance\"} != 1", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} => {{addr}}", + "range": true, + "refId": "A" + } + ], + "title": "Storage reachability ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows network usage between vminserts and clients, such as vmagent, Prometheus, or any other client pushing metrics to vminsert.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1369 + }, + "id": 208, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "read from client", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "write to client", + "range": true, + "refId": "B" + } + ], + "title": "Network usage: clients ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows network usage between vminserts and vmstorages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1369 + }, + "id": 209, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcpdialer_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "read from vmstorage", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "write to vmstorage", + "range": true, + "refId": "B" + } + ], + "title": "Network usage: vmstorage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1377 + }, + "id": 88, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(increase(vm_rows_per_insert_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "range": true, + "refId": "A" + } + ], + "title": "Rows per insert ($instance)", + "type": "timeseries" + } + ], + "title": "vminsert ($instance)", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 194, + "panels": [ + { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 1394 + }, + "id": 198, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.", + "mode": "markdown" + }, + "pluginVersion": "9.2.7", + "title": "", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1396 + }, + "id": 189, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "RSS memory usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "RSS share for memory allocated by the process itself. This share cannot be freed by the OS, so it must be taken into account by OOM killer.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1396 + }, + "id": 190, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "RSS anonymous memory usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1404 + }, + "id": 192, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "CPU usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the approx time needed to reach 100% of disk capacity based on the following params:\n* free disk space (after -storage.minFreeDiskSpaceBytes);\n* row ingestion rate;\n* compression.\n\nNote: this panel doesn't account for deduplication process.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1404 + }, + "id": 196, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "min"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "(vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) \n/ \nignoring(path) (\n rate(vm_rows_added_to_storage_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d])\n * scalar(\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n / \n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n )\n)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Storage full ETA ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1411 + }, + "id": 200, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Disk space usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1411 + }, + "id": 201, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(job, instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{job}}:{{instance}} (indexdb)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"}) by(job, instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{job}}:{{instance}} (datapoints)", + "range": true, + "refId": "B" + } + ], + "title": "Disk space usage by type ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1419 + }, + "id": 203, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, instance, level, location) > 0", + "format": "time_series", + "hide": false, + "interval": "5m", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Logging rate", + "type": "timeseries" + } + ], + "title": "Drilldown", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "VictoriaMetrics - cluster", + "value": "PAF93674D0B4E9963" + }, + "includeAll": false, + "name": "ds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)", + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)", + "refId": "VictoriaMetrics-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vminsert.*\"}, job)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "job_insert", + "options": [], + "query": { + "query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vminsert.*\"}, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmselect.*\"}, job)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "job_select", + "options": [], + "query": { + "query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmselect.*\"}, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmstorage.*\"}, job)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "job_storage", + "options": [], + "query": { + "query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmstorage.*\"}, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{job=~\"$job\"}, instance)", + "includeAll": true, + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(vm_app_version{job=~\"$job\"}, instance)", + "refId": "VictoriaMetrics-instance-Variable-Query" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "filters": [], + "name": "adhoc", + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "VictoriaMetrics - cluster", + "uid": "oS7Bi_0Wz", + "version": 1, + "weekStart": "", + "gnetId": 11176 +} diff --git a/rootfs/usr/share/grafana/api/dashboards/main/drycc_victoriametrics_vmagent.json b/rootfs/usr/share/grafana/api/dashboards/main/drycc_victoriametrics_vmagent.json new file mode 100644 index 0000000..d542fd0 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/main/drycc_victoriametrics_vmagent.json @@ -0,0 +1,7249 @@ +{ + "__inputs": [], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.4.2" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "enable": true, + "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(short_version))", + "hide": true, + "iconColor": "dark-blue", + "name": "version", + "textFormat": "{{short_version}}", + "titleFormat": "Version change" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "enable": true, + "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job, instance)", + "iconColor": "dark-yellow", + "name": "restarts", + "textFormat": "{{job}}:{{instance}} restarted" + } + ] + }, + "description": "Overview for VictoriaMetrics vmagent v1.102.0 or higher", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "icon": "doc", + "tags": [], + "targetBlank": true, + "title": "vmagent wiki", + "tooltip": "", + "type": "link", + "url": "https://docs.victoriametrics.com/vmagent/" + }, + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "Found a bug?", + "type": "link", + "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues" + }, + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "New releases", + "type": "link", + "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 105, + "panels": [], + "title": "Stats", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) scraped from configured targets.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 103, + "links": [ + { + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_scraped_samples_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Samples scraped/s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of ingested [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 102, + "links": [ + { + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmagent_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Samples ingested/s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of targets scraped per second.", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 134, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_scrapes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) ", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Targets scraped/s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows total number of all configured scrape targets in state `up` or `down`.\n\nSee `http://vmagent-host:8429/targets` to get list of all targets. \n", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "down" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 72, + "links": [ + { + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"up\"})", + "interval": "", + "legendFormat": "up", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"down\"})", + "hide": false, + "interval": "", + "legendFormat": "down", + "range": true, + "refId": "B" + } + ], + "title": "Scrape targets", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows number of generated error messages in logs over last 30m. Non-zero value may be a sign of connectivity or missconfiguration errors.", + "fieldConfig": { + "defaults": { + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 16, + "links": [ + { + "targetBlank": true, + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(increase(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[30m]))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Log errors (30m)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Persistent queue size shows size of pending samples in bytes which hasn't been flushed to remote storage yet. \nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 10485760 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 56, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(vm_persistentqueue_bytes_pending{job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Persistent queue size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "minWidth": 50 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Count" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 101, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 13, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(up{job=~\"$job\", instance=~\"$instance\"}) by (job)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "__auto", + "refId": "A" + } + ], + "title": "Uptime", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 24, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "refId": "A" + } + ], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows in/out [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) rate including push and pull models. \n\nThe out-rate could be different to in-rate because of replication or additional timeseries added by vmagent for every scraped target.\n\nClick on the line and choose Drilldown to show CPU usage per instance\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "title": "Drilldown", + "url": "/d/G7Z9GzMGz?viewPanel=123&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/out .*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate({__name__=~\"vmagent_rows_inserted_total|vm_promscrape_scraped_samples_sum\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "hide": false, + "interval": "", + "legendFormat": "in {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmagent_remotewrite_block_size_rows_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "interval": "", + "legendFormat": "out {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Samples rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the persistent queue size of pending samples in bytes >2MB which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\nClick on the line and choose Drilldown to show the persistent queue size per instance.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/G7Z9GzMGz?viewPanel=125&var-url=${__field.labels.url}&var-ds=$ds&var-instance=$instance&var-job=$job&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 17, + "links": [ + { + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vmagent_remotewrite_pending_data_bytes{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}) by (job, url) > 2e6", + "interval": "", + "legendFormat": "{{job}} => {{url}}", + "range": true, + "refId": "A" + } + ], + "title": "Persistent queue size ($instance) to ($url)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 107, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, level) > 0", + "format": "time_series", + "hide": false, + "interval": "5m", + "intervalFactor": 1, + "legendFormat": "{{job}} - {{level}}", + "range": true, + "refId": "A" + } + ], + "title": "Logging rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows rate of dropped [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) from persistent queue. vmagent drops samples from queue if in-memory and on-disk queues are full and it is unable to flush them to remote storage.\nThe max size of on-disk queue is configured by `-remoteWrite.maxDiskUsagePerURL` flag.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 49, + "links": [ + { + "targetBlank": true, + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_persistentqueue_bytes_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0", + "interval": "", + "legendFormat": "{{ path }}", + "range": true, + "refId": "A" + } + ], + "title": "Persistent queue dropped rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of requests served by vmagent HTTP server.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 15, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmagent_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, path) > 0", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Requests rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Errors rate shows rate for multiple metrics that track possible errors in vmagent, such as network or parsing errors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 69, + "links": [ + { + "targetBlank": true, + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmagent_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, protocol) > 0", + "interval": "", + "legendFormat": "requests:{{protocol}} ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_protoparser_read_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", + "interval": "", + "legendFormat": "parse: {{type}} ({{job}})", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_ingestserver_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", + "interval": "", + "legendFormat": "ingest: {{type}} ({{job}})", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_protoparser_unmarshal_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", + "interval": "", + "legendFormat": "unmarshal: {{type}} ({{job}})", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_dial_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "interval": "", + "legendFormat": "scrape dial ({{job}})", + "range": true, + "refId": "E" + } + ], + "title": "Errors rate ($instance)", + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 45, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Click on the line and choose Drilldown to show CPU usage per instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/G7Z9GzMGz?viewPanel=119&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 109, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Percentage of used RSS memory (resident).\nThe RSS memory shows the amount of memory recently accessed by the application. It includes anonymous memory and data from recently accessed files (aka page cache).\nThe application's performance will significantly degrade when memory usage is close to 100%.\n\nClick on the line and choose Drilldown to show memory usage per instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/G7Z9GzMGz?viewPanel=117&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 111, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RSS memory % usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of bytes read/write from the storage layer when vmagent has to buffer data on disk or read already buffered data.\n\nClick on the line and choose Drilldown to show CPU usage per instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Drilldown", + "url": "/d/G7Z9GzMGz?viewPanel=121&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "read" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 81, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "read {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Disk writes/reads ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Network usage shows the bytes rate for data accepted by vmagent and pushed via remotewrite protocol.\nDiscrepancies are possible because of different protocols used for ingesting, scraping and writing data.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) * 8 \n+ sum(rate(vm_promscrape_conn_bytes_read_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) * 8", + "interval": "", + "legendFormat": "in {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmagent_remotewrite_conn_bytes_written_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) * 8", + "interval": "", + "legendFormat": "out {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Network usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Panel shows the percentage of open file descriptors in the OS per instance.\nReaching the limit of open files (100%) can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 5, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 83, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(\n max_over_time(process_open_fds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_max_fds{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Open FDs usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 39, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(max_over_time(go_goroutines{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/troubleshooting/ ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 135, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(\n rate(go_gc_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n ) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU spent on GC ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 149, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Go scheduling latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 41, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(max_over_time(process_num_threads{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Threads ($instance)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "refId": "A" + } + ], + "title": "Resource usage", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 94, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows top 10 jobs by the number of new series registered by vmagent over the 5min range. These jobs generate the most of the churn rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 92, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk_max(10, sum(sum_over_time(scrape_series_added[5m])) by (job)) > 0", + "interval": "", + "legendFormat": "{{ job }}", + "range": true, + "refId": "A" + } + ], + "title": "Top 10 jobs by unique samples", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows top 10 instances by the number of new series registered by vmagent over the 5min range. These instances generate the most of the churn rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 95, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk(10, sum(sum_over_time(scrape_series_added[5m])) by (instance)) > 0", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Top 10 instances by unique samples", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows write saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 98, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(rate(vm_persistentqueue_write_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Persistent queue write saturation ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows read saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with reading data from the disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 99, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(rate(vm_persistentqueue_read_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Persistent queue read saturation ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of dropped data blocks in cases when remote storage replies with `400 Bad Request` and `409 Conflict` HTTP responses.\n\nSee https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 79, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vmagent_remotewrite_packets_dropped_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) by(job, url) > 0", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Data blocks dropped ($instance) to ($url)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of dropped [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) due to relabeling. \nMetric tracks drops for `-remoteWrite.relabelConfig` configuration only.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 18, + "links": [ + { + "targetBlank": true, + "title": "Relabeling", + "url": "https://docs.victoriametrics.com/vmagent/#relabeling" + } + ], + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmagent_remotewrite_relabel_metrics_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, url) > 0", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Rows dropped by relabeling ($instance) to ($url)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of parsed datapoints from write or scrape requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 127, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_protoparser_rows_read_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", + "interval": "", + "legendFormat": "{{ type }} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Datapoints rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Tracks the rate of dropped invalid rows because of errors while unmarshaling write requests. The exact errors messages will be printed in logs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 50, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_rows_invalid_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", + "interval": "", + "legendFormat": "{{type}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Invalid datapoints rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 129, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "job" + } + ] + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(flag{is_set=\"true\", job=~\"$job\", instance=~\"$instance\"}) by(job, instance, name, value)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Non-default flags", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "instance": { + "aggregations": [] + }, + "job": { + "aggregations": [], + "operation": "groupby" + }, + "name": { + "aggregations": [], + "operation": "groupby" + }, + "value": { + "aggregations": [], + "operation": "groupby" + } + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 150, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)", + "format": "time_series", + "instant": false, + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "Restarts ($job)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Troubleshooting", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 28, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 48, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"up\"}) by(job, type) > 0", + "format": "time_series", + "interval": "", + "legendFormat": "{{type}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Scrape targets UP(By Type)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 76, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"down\"}) by(job, type) > 0", + "format": "time_series", + "interval": "", + "legendFormat": "{{type}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Scrape targets DOWN(By Type)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 60 + }, + "id": 132, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vm_promscrape_scrape_pool_targets{job=~\"$job\", instance=~\"$instance\", status=\"up\"}) by(job, scrape_job) > 0", + "format": "time_series", + "interval": "", + "legendFormat": "{{job}}: {{scrape_job}}", + "range": true, + "refId": "A" + } + ], + "title": "Scrape targets UP(By Job)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 60 + }, + "id": 133, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vm_promscrape_scrape_pool_targets{job=~\"$job\", instance=~\"$instance\", status=\"down\"}) by(job, scrape_job) > 0", + "format": "time_series", + "interval": "", + "legendFormat": "{{job}}: {{scrape_job}}", + "range": true, + "refId": "A" + } + ], + "title": "Scrape targets DOWN(By Job)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of scrapes per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 20, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_promscrape_scrapes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Scrape rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of datapoints scraped per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 126, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_promscrape_scraped_samples_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Scraped datapoints rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 46, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_response_size_bytes_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)", + "format": "time_series", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Scrape response size 0.99 quantile ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 75 + }, + "id": 148, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)", + "format": "time_series", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Scrape duration 0.99 quantile ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 31, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_scrapes_failed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "interval": "", + "legendFormat": "scrapes failed ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_scrapes_timed_out_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "interval": "", + "legendFormat": "timeouts ({{job}})", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_scrapes_gunzip_failed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "interval": "", + "legendFormat": "gunzip fails ({{job}})", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_dial_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "interval": "", + "legendFormat": "dial fails ({{job}})", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_max_scrape_size_exceeded_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "hide": false, + "interval": "", + "legendFormat": "max scrape size exceeded ({{job}})", + "range": true, + "refId": "E" + } + ], + "title": "Scrape fails ($instance)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "refId": "A" + } + ], + "title": "Scraping", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 71, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of write requests served by ingestserver (UDP, TCP connections) and HTTP server.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 73, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vm_ingestserver_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type, net) > 0", + "interval": "", + "legendFormat": "{{net}}: {{ type }} ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vmagent_http_requests_total{job=~\"$job\", instance=~\"$instance\", protocol!=\"\"}[$__rate_interval])) by(job, protocol) > 0", + "interval": "", + "legendFormat": "{{ protocol }}: http ({{job}})", + "range": true, + "refId": "B" + } + ], + "title": "Requests rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of rows ingested in vmagent via push protocols.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 131, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vmagent_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0", + "interval": "", + "legendFormat": "{{ type }} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Rows rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows how many concurrent inserts are taking place.\n\nIf the number of concurrent inserts hitting the `limit` or is close to the `limit` constantly - it might be a sign of a resource shortage.\n\n If vmagent's CPU usage and remote write connection saturation are at normal level, it might be that `-maxConcurrentInserts` cmd-line flag need to be increased.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 130, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max_over_time(vm_concurrent_insert_current{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(vm_concurrent_insert_capacity{job=~\"$job\", instance=~\"$instance\"}) by(job)", + "interval": "", + "legendFormat": "limit ({{job}})", + "range": true, + "refId": "B" + } + ], + "title": "Concurrent inserts ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of write errors in ingestserver (UDP, TCP connections) and HTTP server.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 77, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "sum(rate(vm_ingestserver_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(type, net) > 0", + "interval": "", + "legendFormat": "{{ type }} ({{net}})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": true, + "expr": "sum(rate(vmagent_http_request_errors_total{job=~\"$job\", instance=~\"$instance\", protocol!=\"\"}[$__rate_interval])) by(protocol) > 0", + "interval": "", + "legendFormat": "{{ protocol }} (http)", + "refId": "B" + } + ], + "title": "Error rate ($instance)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "refId": "A" + } + ], + "title": "Ingestion", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 136, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of matched samples by the aggregation rule. \n\nThe more samples is matched, the more work this aggregation rule does. The matching rule is specified via `match` param.\n\nSee more details in [stream aggregation config](https://docs.victoriametrics.com/stream-aggregation/#stream-aggregation-config). ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 146, + "options": { + "legend": { + "calcs": ["min", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_streamaggr_matched_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Matched samples ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The rate of ignored samples during aggregation. \nStream aggregation will drop samples with NaN values, or samples with too old timestamps. See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 143, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_streamaggr_ignored_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0) without (instance, pod)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Ignored samples ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of produced samples by the aggregation rule. \n\nNumber of produced samples depend on params like `by`, `without`, `interval`, etc.\n\nSee more details in [stream aggregation config](https://docs.victoriametrics.com/stream-aggregation/#stream-aggregation-config). ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 147, + "options": { + "legend": { + "calcs": ["min", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_streamaggr_output_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Produced samples ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows events when deduplication or aggregation couldn't be finished in the configured interval. Such events may result into bad accuracy of produced data.\n\nPossible solutions:\n* increase interval; \n* use match filter matching smaller number of series;\n* reduce samples ingestion rate to stream aggregation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": -5, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 139, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0", + "instant": false, + "legendFormat": "aggregation: {{url}} ({{job}}): {{path}}:{{position}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0", + "hide": false, + "instant": false, + "legendFormat": "deduplication: {{url}} ({{job}}): {{path}}:{{position}}", + "range": true, + "refId": "B" + } + ], + "title": "Flush timeouts ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the max lag between samples timestamps within one batch passed to the aggregation input.\n\nLower is better.\n\nToo high lag or lag exceeding the interval might be a sign that data was delayed before aggregation or resource insufficiency on aggregator. Samples with high lag may affect accuracy of aggregation.\n\nSee https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 142, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Samples lag 0.99 quantile ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The 99th percentile of avg flush duration for the aggregated data. \n\nSmaller is better.\n\nAggregation can produce incorrect results ff flush duration exceeds configured deduplication interval. See \"Flush Timeouts\" panel.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 137, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, rate(vm_streamaggr_dedup_flush_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Dedup flush duration 0.99 quantile ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the size of Label Compressor in number of entries.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/bytes.*/" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 141, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(vm_streamaggr_labels_compressor_items_count{job=~\"$job\",instance=~\"$instance\"}) by(job, instance)", + "hide": false, + "instant": false, + "legendFormat": "items: {{instance}} ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(vm_streamaggr_labels_compressor_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)", + "hide": false, + "instant": false, + "legendFormat": "bytes: {{instance}} ({{job}})", + "range": true, + "refId": "B" + } + ], + "title": "Labels compressor ($instance)", + "type": "timeseries" + } + ], + "title": "Streaming aggregation", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 58, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of requests to configured remote write endpoints by url and status code.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 60, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vmagent_remotewrite_requests_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) by(job, url, status_code) > 0", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Requests rate ($instance) to ($url)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the global rate for number of written bytes via remote write connections.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 66, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vmagent_remotewrite_conn_bytes_written_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Bytes write rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows requests retry rate by url. Number of retries is unlimited but protected with delays up to 1m between attempts.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 61, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(vmagent_remotewrite_retries_count_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) by(url) > 0", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Retry rate ($instance) to ($url)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows current number of established connections to remote write endpoints.\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 65, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(max_over_time(vmagent_remotewrite_conns{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Connections ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the current limit usage of unique series over an hourly period. Vmagent will start to drop series once the limit is reached.\n\nPlease note, panel will be blank if `remoteWrite.maxHourlySeries` is not set.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 88, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n vmagent_hourly_series_limit_current_series{job=~\"$job\", instance=~\"$instance\"} \n / \n vmagent_hourly_series_limit_max_series{job=~\"$job\", instance=~\"$instance\"}\n ) by(job) * 100", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Hourly series limit", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows saturation of every connection to remote storage. If the threshold of 90% is reached, then the connection is saturated (busy or slow) by more than 90%, so vmagent won't be able to keep up and can start buffering data. \n\nThis usually means that `-remoteWrite.queues` command-line flag must be increased in order to increase the number of connections per each remote storage.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 84, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n rate(vmagent_remotewrite_send_duration_seconds_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])\n /\n vmagent_remotewrite_queues{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}\n) by(job, url)", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Remote write connection saturation ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the current limit usage of unique series over a daily period. Vmagent will start to drop series once the limit is reached.\n\nPlease note, panel will be blank if `remoteWrite.maxDailySeries` is not set.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent" + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 90, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\n vmagent_daily_series_limit_current_series{job=~\"$job\",instance=~\"$instance\"} \n / \n vmagent_daily_series_limit_max_series{job=~\"$job\",instance=~\"$instance\"}\n) by(job)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Daily series limit", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "refId": "A" + } + ], + "title": "Remote write", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 113, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 115, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.", + "mode": "markdown" + }, + "pluginVersion": "10.4.2", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 119, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "CPU usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 117, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "RSS memory usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the persistent queue size of pending samples in bytes which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 125, + "links": [ + { + "title": "Troubleshooting", + "url": "https://docs.victoriametrics.com/vmagent/#troubleshooting" + } + ], + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(vmagent_remotewrite_pending_data_bytes{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}) by (instance, url)", + "interval": "", + "legendFormat": "{{instance}} => {{url}}", + "range": true, + "refId": "A" + } + ], + "title": "Persistent queue size ($instance) to ($url)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows in/out samples rate including push and pull models. \n\nThe out-rate could be different to in-rate because of replication or additional timeseries added by vmagent for every scraped target.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/out .*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 123, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_promscrape_scraped_samples_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)\n+ sum(rate(vmagent_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", + "hide": false, + "interval": "", + "legendFormat": "in {{instance}} {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmagent_remotewrite_block_size_rows_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", + "interval": "", + "legendFormat": "out {{instance}} {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Samples rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of bytes read/write from the storage layer when vmagent has to buffer data on disk or read already buffered data.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "read" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 121, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "read {{instance}} {{job}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job,instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{instance}} {{job}}", + "range": true, + "refId": "B" + } + ], + "title": "Disk writes/reads ($instance)", + "type": "timeseries" + } + ], + "title": "Drilldown", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": ["vmagent", "victoriametrics"], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "VictoriaMetrics - cluster", + "value": "PAF93674D0B4E9963" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "ds", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": "", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{version=~\"^vmagent.*\"}, job)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(vm_app_version{version=~\"^vmagent.*\"}, job)", + "refId": "VictoriaMetrics-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{job=~\"$job\"}, instance)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(vm_app_version{job=~\"$job\"}, instance)", + "refId": "VictoriaMetrics-instance-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vmagent_remotewrite_requests_total{job=~\"$job\", instance=~\"$instance\"}, url)", + "description": "The remote write URLs", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "url", + "options": [], + "query": { + "query": "label_values(vmagent_remotewrite_requests_total{job=~\"$job\", instance=~\"$instance\"}, url)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "filters": [], + "hide": 0, + "name": "adhoc", + "skipUrlSync": false, + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "VictoriaMetrics - vmagent", + "uid": "G7Z9GzMGz", + "version": 1, + "weekStart": "", + "gnetId": 12683 +} diff --git a/rootfs/usr/share/grafana/api/dashboards/main/kubernetes.json b/rootfs/usr/share/grafana/api/dashboards/main/kubernetes.json new file mode 100644 index 0000000..34566cf --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/main/kubernetes.json @@ -0,0 +1,5926 @@ +{ + "annotations": { + "list": [ + { + "$$hashKey": "object:247", + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "【English version】2025.01.25 Updates，kubernetesComprehensive display of resources！Includes K8S Overall Resource Overview、Microservices Resource Details、Pod Resource Details andK8SNetwork Bandwidth，Optimization metrics。https://grafana.com/orgs/starsliao/dashboards", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "icon": "bolt", + "tags": [], + "targetBlank": true, + "title": "Update", + "tooltip": "See more dashboards", + "type": "link", + "url": "https://grafana.com/orgs/starsliao/dashboards" + }, + { + "$$hashKey": "object:831", + "icon": "question", + "tags": ["node_exporter"], + "targetBlank": true, + "title": "GitHub", + "tooltip": "View more dashboards", + "type": "link", + "url": "https://github.com/starsliao" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 54, + "panels": [], + "title": "Node Resource Overview：Selected Nodes:【$Node】", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 44, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Memory Utilization", + "refId": "C" + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Memory Request Rate", + "refId": "A" + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Memory Limit Rate", + "refId": "B", + "step": 10 + } + ], + "title": "Node Memory Ratio", + "type": "bargauge" + }, + { + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 45, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m])) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "CPU Utilization Rate", + "refId": "C" + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "CPU Request Rate", + "refId": "A" + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "CPU Limit Rate", + "refId": "B" + } + ], + "title": "Node CPU Ratio", + "type": "bargauge" + }, + { + "description": "Number of cluster nodes，Nodes POD Number of，NodesPODUpper Limit", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 1000 + }, + { + "color": "red", + "value": 2000 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 8, + "y": 1 + }, + "id": 74, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "count(kube_node_info{origin_prometheus=~\"$origin_prometheus\"})", + "instant": true, + "interval": "", + "legendFormat": "Number of nodes", + "refId": "A" + }, + { + "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",created_by_kind!~\"|Job\",node=~\"^$Node$\"})", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Pod Number of nodes", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"pods\", unit=\"integer\",node=~\"^$Node$\"})", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Upper limit Pod", + "refId": "C" + } + ], + "title": "Nodes with Pod", + "type": "bargauge" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "color-text" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Space" + }, + "properties": [ + { + "id": "custom.width", + "value": 59 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pod" + }, + "properties": [ + { + "id": "custom.width", + "value": 21 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SVC" + }, + "properties": [ + { + "id": "custom.width", + "value": 7 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Microservices" + }, + "properties": [ + { + "id": "custom.width", + "value": 4 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Configuration" + }, + "properties": [ + { + "id": "custom.width", + "value": 16 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Passwords" + }, + "properties": [ + { + "id": "custom.width", + "value": 33 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 11, + "y": 1 + }, + "id": 51, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": true + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Microservices" + } + ] + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\"}) by (namespace)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "count(kube_service_info{origin_prometheus=~\"$origin_prometheus\"}) by(namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "count by (namespace)({__name__=~\"kube_deployment_metadata_generation|kube_daemonset_metadata_generation|kube_statefulset_metadata_generation\",origin_prometheus=~\"$origin_prometheus\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "refId": "D" + }, + { + "editorMode": "code", + "expr": "count(kube_configmap_info{origin_prometheus=~\"$origin_prometheus\"}) by(namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "configmap", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "count(kube_secret_info{origin_prometheus=~\"$origin_prometheus\"}) by(namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "secret", + "refId": "E" + } + ], + "title": "Namespace Resource Statistics", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "namespace" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true + }, + "includeByName": {}, + "indexByName": { + "Time 1": 2, + "Time 2": 4, + "Time 3": 6, + "Value #A": 3, + "Value #C": 5, + "Value #D": 1, + "namespace": 0 + }, + "renameByName": { + "Time 1": "", + "Time 2": "", + "Value #A": "Pod", + "Value #B": "Configuration", + "Value #C": "SVC", + "Value #D": "Microservices", + "Value #E": "Passwords", + "namespace": "Spaces" + } + } + } + ], + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 32, + "options": { + "legend": { + "calcs": ["max"], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 200 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum (irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m]))*8", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Receive", + "metric": "network", + "range": true, + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "sum (irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m]))*8", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Send", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "title": "$NameSpace：Network Overview（Associable nodes and namespaces）", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "max": 2000000000000, + "min": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 100000000000 + }, + { + "color": "red", + "value": 2000000000000 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 71, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "exemplar": true, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Total Memory", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Usage", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Requests", + "refId": "D" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Limits", + "refId": "B" + } + ], + "title": "Node Memory Information", + "type": "bargauge" + }, + { + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "max": 500, + "min": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 500 + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 72, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "exemplar": true, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Total Cores", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",id=\"/\",node=~\"^$Node$\"}[2m]))", + "instant": true, + "interval": "", + "legendFormat": "Usage", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Requests", + "refId": "D" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Limit", + "refId": "B" + } + ], + "title": "Node CPU Number of cores", + "type": "bargauge" + }, + { + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "max": 8000000000000, + "min": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 5000000000000 + }, + { + "color": "red", + "value": 10000000000000 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Utilization" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "max" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "thresholds", + "value": { + "mode": "percentage", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 90 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "decimals", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 8, + "y": 5 + }, + "id": 73, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"}) / sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Utilization Rate", + "refId": "C" + }, + { + "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Usage", + "refId": "A" + }, + { + "expr": "sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})", + "instant": true, + "interval": "", + "legendFormat": "Total", + "refId": "B" + } + ], + "title": "Node Storage Information", + "type": "bargauge" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Anomaly.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 88, + "maxPerRow": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "repeat": "origin_prometheus", + "repeatDirection": "v", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "count({__name__=~\"kube_deployment_metadata_generation|kube_daemonset_metadata_generation|kube_statefulset_metadata_generation\",origin_prometheus=~\"$origin_prometheus\"})", + "hide": false, + "instant": true, + "legendFormat": "Workload", + "range": false, + "refId": "F" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\"})", + "hide": false, + "instant": true, + "legendFormat": "Total Pod", + "range": false, + "refId": "E" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "count by(key,origin_prometheus)(kube_node_spec_taint{origin_prometheus=~\"$origin_prometheus\",key=~\"node.kubernetes.io.*\"})", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": "{{key}}", + "range": false, + "refId": "D" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "count by(origin_prometheus)(kube_node_info{origin_prometheus=~\"$origin_prometheus\"})", + "hide": false, + "instant": true, + "legendFormat": "Total Nodes", + "range": false, + "refId": "C" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "count by(origin_prometheus)(kube_node_info{origin_prometheus=~\"$origin_prometheus\"}) - count by(origin_prometheus)(kube_node_spec_taint{origin_prometheus=~\"$origin_prometheus\",key!~\"node.kubernetes.io.*\"})", + "hide": false, + "instant": true, + "legendFormat": "Normal Node", + "range": false, + "refId": "B" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "count by(key,origin_prometheus)(kube_node_spec_taint{origin_prometheus=~\"$origin_prometheus\",key!~\"node.kubernetes.io.*\"})", + "hide": false, + "instant": true, + "legendFormat": "{{key}}", + "range": false, + "refId": "A" + } + ], + "title": "", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "(node.kubernetes.io/)(.*)", + "renamePattern": "Abnormal:$2" + } + } + ], + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total Memory" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 11 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", + "instant": false, + "interval": "", + "legendFormat": "Total Memory", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})", + "instant": false, + "interval": "", + "legendFormat": "Usage", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", + "hide": true, + "instant": false, + "interval": "", + "legendFormat": "Requests", + "refId": "D" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", + "hide": true, + "instant": false, + "interval": "", + "legendFormat": "Limit", + "refId": "B" + } + ], + "title": "Memory Usage【$Node】", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total Cores" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 11 + }, + "id": 80, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", + "instant": false, + "interval": "", + "legendFormat": "Total Cores", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",id=\"/\",node=~\"^$Node$\"}[2m]))", + "instant": false, + "interval": "", + "legendFormat": "Usage", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", + "hide": true, + "instant": false, + "interval": "", + "legendFormat": "Requests", + "refId": "D" + }, + { + "exemplar": true, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", + "hide": true, + "instant": true, + "interval": "", + "legendFormat": "Limit", + "refId": "B" + } + ], + "title": "CPU Used Cores【$Node】", + "type": "timeseries" + }, + { + "description": "Number of Cluster Nodes，Nodes POD Number of，Nodes POD Upper Limit", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Upper Limit Pod" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Number of nodes" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.drawStyle", + "value": "points" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.pointSize", + "value": 3 + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 11 + }, + "id": 81, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "exemplar": true, + "expr": "count(kube_node_info{origin_prometheus=~\"$origin_prometheus\"})", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Number of nodes", + "refId": "A" + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",created_by_kind!~\"|Job\",node=~\"^$Node$\"})", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Pod Number", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"pods\", unit=\"integer\",node=~\"^$Node$\"})", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Upper Limit Pod", + "refId": "C" + } + ], + "title": "Pod Number and nodes【$Node】", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Total number of cores.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 75, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "exemplar": true, + "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m]))by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)*100", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{node}}", + "refId": "I" + } + ], + "title": "$Node：Nodes CPU Breakdown", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 76, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "exemplar": true, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)*100", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{node}}", + "refId": "I" + } + ], + "title": "$Node：Node Memory Breakdown", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "←Inflow/Outflow→", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Inflow.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 78, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "exemplar": true, + "expr": "sum (irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\"}[2m]))by (node) *8", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Inflow:{{node}}", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "exemplar": true, + "expr": "sum (irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\"}[2m]))by (node) *8", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Outflows:{{node}}", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "title": "$Node：Node Network Overview", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "CPU Limitations" + }, + "properties": [ + { + "id": "custom.width", + "value": 76 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 71 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Limits" + }, + "properties": [ + { + "id": "custom.width", + "value": 74 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Disk Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 74 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*%" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + }, + { + "id": "custom.width", + "value": 85 + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "(Memory Usage|Total Memory|Memory Request|Memory Limit|Disk Usage|Disk Total)" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nodes" + }, + "properties": [ + { + "id": "custom.width", + "value": 96 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Requests" + }, + "properties": [ + { + "id": "custom.width", + "value": 76 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CPU Requests" + }, + "properties": [ + { + "id": "custom.width", + "value": 75 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "(CPUTotal|MemoryTotal|DiskTotal|PodLimit)" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PodCap" + }, + "properties": [ + { + "id": "custom.width", + "value": 66 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "CPU Core Usage$|Memory Usage$|Disk Usage$|Pod Number of" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Total/" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pod Number of" + }, + "properties": [ + { + "id": "custom.width", + "value": 58 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CPUTotal Cores" + }, + "properties": [ + { + "id": "custom.width", + "value": 69 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Memory" + }, + "properties": [ + { + "id": "custom.width", + "value": 75 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Disk" + }, + "properties": [ + { + "id": "custom.width", + "value": 74 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CPU Core Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 74 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Usage%" + }, + "properties": [ + { + "id": "custom.width", + "value": 102 + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 52, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Memory Usage%" + } + ] + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "exemplar": false, + "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",created_by_kind!~\"|Job\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "pod Number", + "refId": "A" + }, + { + "exemplar": false, + "expr": "kube_node_status_condition{origin_prometheus=~\"$origin_prometheus\",status=\"true\",node=~\"^$Node$\"} == 1", + "format": "table", + "hide": true, + "instant": true, + "interval": "", + "legendFormat": "Status", + "refId": "B" + }, + { + "exemplar": false, + "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m])) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "I" + }, + { + "exemplar": false, + "expr": "kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"} - 0", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + }, + { + "exemplar": false, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "J" + }, + { + "exemplar": false, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"}) by (node) - 0", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "G" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "H" + }, + { + "exemplar": false, + "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "K" + }, + { + "exemplar": false, + "expr": "sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"}) by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "L" + }, + { + "exemplar": false, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Memory Usage%", + "refId": "M" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Memory Requests%", + "refId": "N" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Memory Limit%", + "refId": "O" + }, + { + "exemplar": false, + "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m]))by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "CPU Usage%", + "refId": "P" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "CPU Requests%", + "refId": "Q" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Memory Limit%", + "refId": "R" + }, + { + "exemplar": false, + "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})by (node) / sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Disk Usage%", + "refId": "S" + }, + { + "exemplar": false, + "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"pods\", unit=\"integer\",node=~\"^$Node$\"})by (node)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "PodCaps", + "refId": "T" + } + ], + "title": "$Node：Node Information Detail", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 10": true, + "Time 11": true, + "Time 12": true, + "Time 13": true, + "Time 14": true, + "Time 15": true, + "Time 16": true, + "Time 17": true, + "Time 18": true, + "Time 19": true, + "Time 2": true, + "Time 20": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true, + "Time 8": true, + "Time 9": true, + "Value #B": true, + "Value #E": false, + "Value #F": false, + "__name__": true, + "app_kubernetes_io_name": true, + "app_kubernetes_io_name 1": true, + "app_kubernetes_io_name 2": true, + "app_kubernetes_io_name 3": true, + "app_kubernetes_io_version": true, + "app_kubernetes_io_version 1": true, + "app_kubernetes_io_version 2": true, + "app_kubernetes_io_version 3": true, + "condition": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "instance 3": true, + "job": true, + "job 1": true, + "job 2": true, + "job 3": true, + "k8s_namespace": true, + "k8s_namespace 1": true, + "k8s_namespace 2": true, + "k8s_namespace 3": true, + "k8s_sname": true, + "k8s_sname 1": true, + "k8s_sname 2": true, + "k8s_sname 3": true, + "origin_prometheus": true, + "origin_prometheus 1": true, + "origin_prometheus 2": true, + "origin_prometheus 3": true, + "resource": true, + "status": true, + "unit": true + }, + "includeByName": {}, + "indexByName": { + "Time": 22, + "Value #A": 2, + "Value #C": 6, + "Value #D": 8, + "Value #E": 16, + "Value #F": 17, + "Value #G": 18, + "Value #H": 19, + "Value #I": 7, + "Value #J": 9, + "Value #K": 11, + "Value #L": 10, + "Value #M": 4, + "Value #N": 13, + "Value #O": 15, + "Value #P": 3, + "Value #Q": 12, + "Value #R": 14, + "Value #S": 5, + "Value #T": 1, + "instance": 23, + "job": 24, + "node": 0, + "origin_prometheus": 25, + "resource": 20, + "unit": 21 + }, + "renameByName": { + "Value #A": "Pod Number of", + "Value #C": "CPU Total Cores", + "Value #D": "Total Memory", + "Value #E": "CPU Requests", + "Value #F": "CPU Limit", + "Value #G": "Memory Requests", + "Value #H": "Memory Limit", + "Value #I": "CPU Core Usage", + "Value #J": "Memory Usage", + "Value #K": "Disk Usage", + "Value #L": "Disk Total", + "Value #M": "Memory Usage%", + "Value #N": "Memory Requests%", + "Value #O": "Memory Limit%", + "Value #P": "CPU Usage%", + "Value #Q": "CPU Requests%", + "Value #R": "CPU Limit%", + "Value #S": "Disk Usage%", + "Value #T": "Pod Limit", + "condition": "Status", + "node": "Node" + } + } + }, + { + "id": "filterFieldsByName", + "options": {} + } + ], + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "color-background" + }, + "inspect": false + }, + "decimals": 0, + "links": [], + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0 + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "red", + "index": 1 + } + }, + "type": "special" + } + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Utilization" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.width", + "value": 54 + }, + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "decimals" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "from": 75, + "result": { + "color": "semi-dark-red", + "index": 0 + }, + "to": 110 + }, + "type": "range" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mounts Pod Number of" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.width", + "value": 59 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Namespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 58 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PVC" + }, + "properties": [ + { + "id": "custom.width", + "value": 94 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 57 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "custom.width", + "value": 54 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 33 + }, + "id": 92, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Usage Rate" + } + ] + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"})", + "format": "table", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}:{{ persistentvolumeclaim }}", + "metric": "container_memory_usage:sort_desc", + "range": false, + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "min by (namespace,persistentvolumeclaim) (kubelet_volume_stats_available_bytes{origin_prometheus=~\"$origin_prometheus\"}) + max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "metric": "container_memory_usage:sort_desc", + "range": false, + "refId": "B", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"}) /(min by (namespace,persistentvolumeclaim) (kubelet_volume_stats_available_bytes{origin_prometheus=~\"$origin_prometheus\"}) + max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"}))*100", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}:{{ persistentvolumeclaim }}", + "metric": "container_memory_usage:sort_desc", + "range": false, + "refId": "C", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "count by (namespace,persistentvolumeclaim)(kube_pod_spec_volumes_persistentvolumeclaims_info{origin_prometheus=~\"$origin_prometheus\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "metric": "container_memory_usage:sort_desc", + "range": false, + "refId": "D", + "step": 10 + } + ], + "title": "PVC Storage Usage", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value #A": "Usage", + "Value #B": "Total", + "Value #C": "Usage Rate", + "Value #D": "Mount Pod Number", + "namespace": "Namespaces", + "persistentvolumeclaim": "PVC" + } + } + } + ], + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 6, + "y": 33 + }, + "id": 86, + "options": { + "legend": { + "calcs": ["last", "max"], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container !=\"\",container!=\"POD\"}[2m])) by (namespace)>0.5", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ namespace }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "title": "Namespaces CPU Usage kernel(>0.5)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 33 + }, + "id": 85, + "options": { + "legend": { + "calcs": ["last", "max"], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container !=\"\",container!=\"POD\"}) by (namespace) > 1*1024*1024*1024", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace} {{ pod }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Namespaces WSS Memory Usage (>1G)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 49, + "panels": [], + "title": "Pod Resource Overview：SelectedPod:【$Pod】", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "displayName": "", + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 80 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Namespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 96 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pod Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 207 + }, + { + "id": "custom.align", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cores Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 71 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Reboot" + }, + "properties": [ + { + "id": "custom.width", + "value": 38 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "decimals" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*%" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.width", + "value": 55 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Restrictions" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nodes" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "Cores in use$|WSS$|RSS$|Survival|Inflow|Outflow" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Container Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 57 + }, + { + "id": "custom.align", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Survival" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used cores" + }, + "properties": [ + { + "id": "custom.width", + "value": 62 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CPU Limits" + }, + "properties": [ + { + "id": "custom.width", + "value": 58 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Limit" + }, + "properties": [ + { + "id": "custom.width", + "value": 68 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Requirements" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "WSS$|RSS$|Memory Requirements$|Memory Limit$|Disk.*$" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "WSS" + }, + "properties": [ + { + "id": "custom.width", + "value": 81 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RSS" + }, + "properties": [ + { + "id": "custom.width", + "value": 74 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CPU Requirements" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Disk Limitations" + }, + "properties": [ + { + "id": "custom.width", + "value": 83 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Disk Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 10737418240 + }, + { + "color": "red", + "value": 16106127360 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "WSS%" + }, + "properties": [ + { + "id": "custom.width", + "value": 77 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/Inflow|Streaming Out/" + }, + "properties": [ + { + "id": "unit", + "value": "binbps" + }, + { + "id": "custom.width", + "value": 80 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 47, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Container Name" + } + ] + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod,node,namespace) / (sum(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}/100000) by (container, pod,node,namespace)) ", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "exemplar": false, + "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "CPUKernel Usage", + "refId": "Q" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "wss%", + "refId": "I" + }, + { + "exemplar": false, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "wss", + "refId": "D" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "rss%", + "refId": "L" + }, + { + "exemplar": false, + "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "rss", + "refId": "K" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "J" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "kube_pod_container_status_restarts_total{\n origin_prometheus=~\"$origin_prometheus\",\n pod=~\"$Pod\",\n container=~\"$Container\",\n container!=\"\",\n container!=\"POD\",\n namespace=~\"$NameSpace\"\n}\n* on (pod) group_left(node) (\n kube_pod_info{\n origin_prometheus=~\"$origin_prometheus\",\n pod=~\"$Pod\",\n node=~\"^$Node$\",\n namespace=~\"$NameSpace\"\n }\n unless on (pod) (\n kube_pod_info{\n origin_prometheus=~\"$origin_prometheus\",\n pod=~\"$Pod\",\n node=~\"^$Node$\",\n namespace=~\"$NameSpace\"\n } offset 1m\n )\n)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "H" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "time() - \n(\n max by(pod, namespace) (kube_pod_created{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\"})\n * on(pod, namespace) group_right(container)\n kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container=~\"$Container\"}\n)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "refId": "R" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "S" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(sum(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "refId": "T" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(sum(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "refId": "U" + } + ], + "title": "$Node：Pod Resource Detail(Associable Nodes)", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 10": true, + "Time 11": true, + "Time 12": true, + "Time 13": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true, + "Time 8": true, + "Time 9": true, + "Value #G": false, + "__name__": true, + "app_kubernetes_io_name": true, + "app_kubernetes_io_name 1": true, + "app_kubernetes_io_name 2": true, + "app_kubernetes_io_version": true, + "app_kubernetes_io_version 1": true, + "app_kubernetes_io_version 2": true, + "container 1": true, + "container 10": true, + "container 11": true, + "container 12": true, + "container 2": true, + "container 3": true, + "container 4": true, + "container 5": true, + "container 6": true, + "container 7": true, + "container 8": true, + "container 9": true, + "created_by_kind": true, + "created_by_name": true, + "host_ip": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "job": true, + "job 1": true, + "job 2": true, + "k8s_namespace": true, + "k8s_namespace 1": true, + "k8s_namespace 2": true, + "k8s_sname": true, + "k8s_sname 1": true, + "k8s_sname 2": true, + "namespace": false, + "namespace 1": true, + "namespace 10": true, + "namespace 11": true, + "namespace 12": false, + "namespace 2": true, + "namespace 3": true, + "namespace 4": true, + "namespace 5": true, + "namespace 6": true, + "namespace 7": true, + "namespace 8": true, + "namespace 9": true, + "node 1": true, + "node 10": true, + "node 11": false, + "node 12": true, + "node 2": true, + "node 3": true, + "node 4": true, + "node 5": true, + "node 6": true, + "node 7": true, + "node 8": true, + "node 9": true, + "origin_prometheus": true, + "origin_prometheus 1": true, + "origin_prometheus 2": true, + "phase": true, + "pod_ip": true, + "priority_class": true, + "uid": true + }, + "includeByName": {}, + "indexByName": { + "Time": 21, + "Value #A": 4, + "Value #B": 16, + "Value #C": 7, + "Value #D": 10, + "Value #E": 17, + "Value #F": 9, + "Value #G": 23, + "Value #H": 14, + "Value #I": 5, + "Value #J": 13, + "Value #K": 11, + "Value #L": 6, + "Value #M": 24, + "Value #N": 25, + "Value #O": 26, + "Value #P": 27, + "Value #Q": 8, + "Value #R": 15, + "Value #S": 12, + "container": 2, + "instance": 18, + "ip": 28, + "job": 19, + "namespace": 1, + "node": 0, + "origin_prometheus": 20, + "pod": 3, + "uid": 22 + }, + "renameByName": { + "Value #A": "CPU%", + "Value #B": "CPU Requirements", + "Value #C": "CPU Limitations", + "Value #D": "WSS", + "Value #E": "Memory Requirements", + "Value #F": "Memory Limit", + "Value #H": "Reboot", + "Value #I": "WSS%", + "Value #J": "Disk Usage", + "Value #K": "RSS", + "Value #L": "RSS%", + "Value #M": "Heap Memory", + "Value #N": "maxHeap", + "Value #O": "Non-Heap", + "Value #P": "maxNon-Heap", + "Value #Q": "Using Cores", + "Value #R": "Survival", + "Value #S": "Disk Limit", + "Value #T": "Inflow", + "Value #U": "Streaming Out", + "container": "Container Name", + "instance": "", + "ip": "POD IP", + "namespace": "Namespace", + "namespace 1": "", + "namespace 12": "Namespace", + "node": "Nodes", + "node 1": "", + "node 11": "Node", + "pod": "Pod Name", + "priority_class": "" + } + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "Node", + "Namespace", + "Container Name", + "Pod Name", + "CPU%", + "WSS%", + "RSS%", + "CPU Restrictions", + "Cores Used", + "Memory Limit", + "WSS", + "RSS", + "Disk Limit", + "Disk Usage", + "Reboot", + "CPUDemand", + "Memory Requirements", + "Inflow", + "Outflow", + "Survival" + ] + } + } + } + ], + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 50 + }, + "id": 58, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "max(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod) / (max(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}/100000) by (container, pod)) * 100", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ pod }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "title": "Pod Containers CPU Utilization (Maximum100%Associable Nodes)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 50 + }, + "id": 27, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "max (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod)/ max(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod) * 100", + "interval": "", + "intervalFactor": 1, + "legendFormat": "WSS：{{ pod }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "max (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod)/ max(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod) * 100", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RSS：{{ pod }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "B", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "(cass_jvm_heap{service=~\"$Container\"} * on (pod_ip) group_right(service) kube_pod_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\"}) / (cass_jvm_heap_max{service=~\"$Container\"} * on (pod_ip) group_right(service) kube_pod_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\"}) * 100", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Heap：{{ pod }}", + "metric": "container_memory_usage:sort_desc", + "refId": "C", + "step": 10 + } + ], + "title": "Pod Container Memory Usage (Associatable Nodes)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 50 + }, + "id": 77, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "max(max(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Inflow:{{ pod}}", + "metric": "network", + "range": true, + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "max(max(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Outflow:{{ pod}}", + "metric": "network", + "range": true, + "refId": "B", + "step": 10 + }, + { + "editorMode": "code", + "expr": "sum(sum(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info) by(pod) *8", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "metric": "network", + "range": true, + "refId": "C", + "step": 10 + }, + { + "editorMode": "code", + "expr": "sum(sum(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info) by(pod) *8", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "metric": "network", + "range": true, + "refId": "D", + "step": 10 + } + ], + "title": "Pod Network bandwidth per second (Associable Nodes)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Limit.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 59 + }, + "id": 82, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "max(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod,node,namespace)", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU Usage：{{ pod }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "max(max(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)) by(container)", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Pod CPU Limitations：{{ container}}", + "metric": "container_cpu", + "refId": "B", + "step": 10 + } + ], + "title": "Pod Containers CPU Core Usage", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Limitations/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 59 + }, + "id": 84, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "max (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "WSS：{{ pod }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "max(max(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)) by(container)", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "PodMemory Limits：{{ container}}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "B", + "step": 10 + } + ], + "title": "Pod Containers WSS Memory Usage (Associable Nodes)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 59 + }, + "id": 83, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "max (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RSS：{{ pod }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Pod Containers RSS Memory Usage (Associable Nodes)", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 61, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*%" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge", + "valueDisplayMode": "color" + } + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Memory Usage$|.*Memory Limits$|.*Memory Requirements$|.*Disk Usage$|.*Disk Limits$" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Namespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 92 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Container Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 187 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total CPU Core Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pod" + }, + "properties": [ + { + "id": "custom.width", + "value": 44 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Average CPU Usage%" + }, + "properties": [ + { + "id": "custom.width", + "value": 116 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Average RSS Memory Usage%" + }, + "properties": [ + { + "id": "custom.width", + "value": 141 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Average WSS Memory Usage%" + }, + "properties": [ + { + "id": "custom.width", + "value": 165 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total CPU Limit" + }, + "properties": [ + { + "id": "custom.width", + "value": 86 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Memory Limit" + }, + "properties": [ + { + "id": "custom.width", + "value": 86 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Limit$/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Memory Usage$|.*Core Usage$/" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RSS Memory Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 107 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total WSS Memory Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 113 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Average Disk Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 96 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 10737418240 + }, + { + "color": "red", + "value": 16106127360 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Average Disk Limit" + }, + "properties": [ + { + "id": "custom.width", + "value": 96 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total CPU Demand" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Memory Demand" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 87, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Average WSS Memory Usage%" + } + ] + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container) / (sum(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}/100000) by (container))", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Total Cores Used", + "refId": "L" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "I" + }, + { + "exemplar": false, + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Average Memory%(RSS)", + "refId": "H" + }, + { + "exemplar": false, + "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Total Memory Usage (RSS) ", + "refId": "K" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "sum by(container) (kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + }, + { + "exemplar": false, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "avg(container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "J" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "count(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by(container,namespace)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "G" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "avg(container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "M" + } + ], + "title": "Microservices (Container Name) Resource Statistics", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 10": true, + "Time 11": true, + "Time 12": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true, + "Time 8": true, + "Time 9": true + }, + "includeByName": {}, + "indexByName": { + "Time": 15, + "Value #A": 3, + "Value #B": 13, + "Value #C": 6, + "Value #D": 9, + "Value #E": 14, + "Value #F": 8, + "Value #G": 2, + "Value #H": 5, + "Value #I": 4, + "Value #J": 12, + "Value #K": 10, + "Value #L": 7, + "Value #M": 11, + "container": 1, + "namespace": 0 + }, + "renameByName": { + "Time 1": "", + "Value #A": "Average CPU Usage%", + "Value #B": "Total CPU Demand", + "Value #C": "Total CPU Restrictions", + "Value #D": "Total WSS Memory Usage", + "Value #E": "Total Memory Requirements", + "Value #F": "Total Memory Limit", + "Value #G": "Pod", + "Value #H": "Average RSS Memory Usage%", + "Value #I": "Average WSS Memory Usage%", + "Value #J": "Average Disk Usage", + "Value #K": "Total RSS Memory Use", + "Value #L": "Total CPU Core Usage", + "Value #M": "Average Disk Limit", + "container": "Container Name", + "namespace": "Namespace" + } + } + }, + { + "id": "filterFieldsByName", + "options": {} + } + ], + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 24, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "exemplar": true, + "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container) / (sum(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}/100000) by (container)) * 100", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ container}}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "title": "Microservices (Container Name) Average CPU Usage (Maximum100%)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 12 + }, + "id": 89, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container) * 100", + "interval": "", + "intervalFactor": 1, + "legendFormat": "WSS：{{ container }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + }, + { + "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container) * 100", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RSS：{{ container }}", + "metric": "container_memory_usage:sort_desc", + "refId": "B", + "step": 10 + } + ], + "title": "Microservice (Container Name) Average Memory Utilization", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 16, + "options": { + "legend": { + "calcs": ["mean", "last", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(sum(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(container) *8", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Inflow:{{ container }}", + "metric": "network", + "range": true, + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(sum(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(container) *8", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Outflow:{{ container }}", + "metric": "network", + "range": true, + "refId": "B", + "step": 10 + }, + { + "expr": "sum (rate (container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "-> {{ pod }}", + "metric": "network", + "refId": "C", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "<- {{ pod }}", + "metric": "network", + "refId": "D", + "step": 10 + } + ], + "title": "Microservice (Container Name) Network bandwidth per second (Associable Nodes)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/CPU Limit.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 91, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU Limitations：{{ container}}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container)", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU Core Usage：{{ container}}", + "metric": "container_cpu", + "refId": "B", + "step": 10 + } + ], + "title": "Microservices (Container Name) Overall CPU Cores Used", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Memory Limit.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 90, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "WSS：{{ container }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "A", + "step": 10 + }, + { + "editorMode": "code", + "expr": "sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Memory Limit：{{ container }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "B", + "step": 10 + }, + { + "editorMode": "code", + "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "RSS：{{ container }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "C", + "step": 10 + } + ], + "title": "Microservices (Container Name) Overall Memory Usage", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 59, + "options": { + "legend": { + "calcs": ["max", "last", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "editorMode": "code", + "expr": "count(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by(container,namespace)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}：{{ container }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Microservice (Container Name) Pod Number", + "type": "timeseries" + } + ], + "title": "Microservices (Container Name) Resource Overview：Selected Microservices:【$Container】", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": ["Prometheus", "Kubernetes"], + "templating": { + "list": [ + { + "current": { + "text": "", + "value": "" + }, + "definition": "label_values(kube_node_info,origin_prometheus)", + "includeAll": false, + "label": "K8S", + "name": "origin_prometheus", + "options": [], + "query": { + "query": "label_values(kube_node_info,origin_prometheus)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 5, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(kube_node_info{origin_prometheus=~\"$origin_prometheus\"},node)", + "includeAll": true, + "label": "Nodes", + "name": "Node", + "options": [], + "query": { + "query": "label_values(kube_node_info{origin_prometheus=~\"$origin_prometheus\"},node)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 5, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(kube_namespace_created{origin_prometheus=~\"$origin_prometheus\"},namespace)", + "includeAll": true, + "label": "Namespace", + "name": "NameSpace", + "options": [], + "query": { + "query": "label_values(kube_namespace_created{origin_prometheus=~\"$origin_prometheus\"},namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 5, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\"},container)", + "includeAll": true, + "label": "Microservice (Container Name)", + "name": "Container", + "options": [], + "query": { + "query": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\"},container)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 5, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container=~\"$Container\"},pod)", + "includeAll": true, + "label": "Pod", + "name": "Pod", + "options": [], + "query": { + "query": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container=~\"$Container\"},pod)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": ["30s", "1m", "5m", "15m", "30m", "1h"] + }, + "timezone": "", + "title": "K8S Dashboard EN 20250125", + "uid": "15661", + "version": 3 +} diff --git a/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_apiserver.json b/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_apiserver.json new file mode 100644 index 0000000..9fda701 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_apiserver.json @@ -0,0 +1,639 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": 12006, + "graphTooltip": 0, + "id": null, + "iteration": 1585691077160, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[5m])) by (verb)", + "legendFormat": "{{verb}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "apiserver request rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "req/s", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(workqueue_queue_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(workqueue_queue_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p50", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "workqueue service time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "service time", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "95th percentile request latency for Kubernetes apiserver component", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\"CONNECT|WATCH\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\"CONNECT|WATCH\"}[5m])) by (le))", + "legendFormat": "p90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\"CONNECT|WATCH\"}[5m])) by (le))", + "legendFormat": "p50 ", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "apiserver request latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": "latency", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(workqueue_work_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.9, sum(rate(workqueue_work_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(workqueue_work_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p50", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "workqueue processing time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "processing time", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 6, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_helper_cache_hit_total[5m]))/sum((rate(etcd_helper_cache_hit_total[5m])+rate(etcd_helper_cache_miss_total[5m])))", + "legendFormat": "ratio", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "etcd helper cache hit ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "etcd request latency (p95)", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p50", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "etcd request latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": "latency", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Kubernetes apiserver", + "uid": "R6abPf9Zz", + "version": 3, + "description": "Latency and cache hit graphs for Kubernetes apiserver. Useful for observing performance of Amazon EKS clusters." +} diff --git a/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_cluster.json b/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_cluster.json new file mode 100644 index 0000000..1d47310 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_cluster.json @@ -0,0 +1,2010 @@ +{ + "id": null, + "title": "Kubernetes cluster monitoring (via Prometheus)", + "description": "Monitors Kubernetes cluster using Prometheus. Shows overall cluster CPU / Memory / Filesystem usage as well as individual pod, containers, systemd services statistics. Uses cAdvisor metrics only.", + "tags": ["kubernetes"], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "200px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)", + "thresholdLine": false + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate (container_network_receive_bytes_total{kubernetes_io_hostname=~\"^$Node$\"}[1m]))", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{kubernetes_io_hostname=~\"^$Node$\"}[1m]))", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O pressure", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Network I/O pressure" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "180px", + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) / sum (machine_memory_bytes{kubernetes_io_hostname=~\"^$Node$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Cluster memory usage", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "180px", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) / sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Cluster CPU usage (1m avg)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "180px", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (container_fs_usage_bytes{device=~\"^/dev/.*$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) / sum (container_fs_limit_bytes{device=~\"^/dev/.*$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Cluster filesystem usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "20%", + "prefix": "", + "prefixFontSize": "20%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (machine_memory_bytes{kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m]))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (container_fs_usage_bytes{device=~\"^/dev/.*$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "id": 14, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum (container_fs_limit_bytes{device=~\"^/dev/.*$\",id=\"/\",kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "showTitle": false, + "title": "Total usage" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": "", + "id": 17, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (pod_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ pod_name }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Pods CPU usage (1m avg)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "showTitle": false, + "title": "Pods CPU usage" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": "", + "id": 23, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{systemd_service_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (systemd_service_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ systemd_service_name }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "System services CPU usage (1m avg)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "System services CPU usage" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": "", + "id": 24, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",name=~\"^k8s_.*\",container_name!=\"POD\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (container_name, pod_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "pod: {{ pod_name }} | {{ container_name }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, name, image)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "container_cpu", + "refId": "B", + "step": 10 + }, + { + "expr": "sum (rate (container_cpu_usage_seconds_total{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, rkt_container_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "container_cpu", + "refId": "C", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Containers CPU usage (1m avg)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Containers CPU usage" + }, + { + "collapse": true, + "editable": true, + "height": "500px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 20, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (id)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ id }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "All processes CPU usage (1m avg)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "showTitle": false, + "title": "All processes CPU usage" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 25, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}) by (pod_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ pod_name }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Pods memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Pods memory usage" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 26, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{systemd_service_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}) by (systemd_service_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ systemd_service_name }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "System services memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "System services memory usage" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 27, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{image!=\"\",name=~\"^k8s_.*\",container_name!=\"POD\",kubernetes_io_hostname=~\"^$Node$\"}) by (container_name, pod_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "pod: {{ pod_name }} | {{ container_name }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + }, + { + "expr": "sum (container_memory_working_set_bytes{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}) by (kubernetes_io_hostname, name, image)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "container_memory_usage:sort_desc", + "refId": "B", + "step": 10 + }, + { + "expr": "sum (container_memory_working_set_bytes{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}) by (kubernetes_io_hostname, rkt_container_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "container_memory_usage:sort_desc", + "refId": "C", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Containers memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Containers memory usage" + }, + { + "collapse": true, + "editable": true, + "height": "500px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 28, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) by (id)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ id }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "All processes memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "All processes memory usage" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 16, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (pod_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> {{ pod_name }}", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (pod_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- {{ pod_name }}", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Pods network I/O (1m avg)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Pods network I/O" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 30, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (container_name, pod_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> pod: {{ pod_name }} | {{ container_name }}", + "metric": "network", + "refId": "B", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (container_name, pod_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- pod: {{ pod_name }} | {{ container_name }}", + "metric": "network", + "refId": "D", + "step": 10 + }, + { + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, name, image)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, name, image)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "network", + "refId": "C", + "step": 10 + }, + { + "expr": "sum (rate (container_network_transmit_bytes_total{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, rkt_container_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "network", + "refId": "E", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (kubernetes_io_hostname, rkt_container_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "network", + "refId": "F", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Containers network I/O (1m avg)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Containers network I/O" + }, + { + "collapse": true, + "editable": true, + "height": "500px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 29, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate (container_network_receive_bytes_total{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (id)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> {{ id }}", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (id)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- {{ id }}", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "All processes network I/O (1m avg)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "All processes network I/O" + } + ], + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "templating": { + "list": [ + { + "allValue": ".*", + "current": {}, + "hide": 0, + "includeAll": true, + "multi": false, + "name": "Node", + "options": [], + "query": "label_values(kubernetes_io_hostname)", + "refresh": 1, + "type": "query" + } + ] + }, + "annotations": { + "list": [] + }, + "refresh": "10s", + "schemaVersion": 12, + "version": 13, + "uid": "315", + "links": [], + "gnetId": 1621 +} diff --git a/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_deployment_metrics.json b/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_deployment_metrics.json new file mode 100644 index 0000000..231f591 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_deployment_metrics.json @@ -0,0 +1,1324 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Monitors Kubernetes deployments in cluster using Prometheus. Shows overall cluster CPU / Memory / Network of deployments, replicas in each deployment. Uses Kube state metrics and cAdvisor metrics ", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 741, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 65 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 4, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}) / sum (machine_memory_bytes{kubernetes_io_hostname=~\"^$Node$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Deployment memory usage", + "type": "gauge" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 65 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 6, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}[1m])) / sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Deployment CPU usage", + "type": "gauge" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 37, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "sum(kube_deployment_status_replicas_available{deployment=~\"^$Deployment$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}) / sum(kube_deployment_status_replicas{deployment=~\"^$Deployment$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}) * 100", + "intervalFactor": 2, + "range": true, + "refId": "A", + "step": 40 + } + ], + "title": "Replicas", + "type": "gauge" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 38, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{namespace=\"$Namespace\",pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\"})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "range": true, + "refId": "A", + "step": 40 + } + ], + "title": "Used", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 39, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": null, + "expr": "sum (container_memory_working_set_bytes{kubernetes_io_hostname=~\"^$Node.*$\"})", + "intervalFactor": 2, + "refId": "A", + "step": 40 + } + ], + "title": "Total", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 5 + }, + "id": 40, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}[1m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "range": true, + "refId": "A", + "step": 40 + } + ], + "title": "Used", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 5 + }, + "id": 41, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": null, + "expr": "sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"})", + "intervalFactor": 2, + "refId": "A", + "step": 40 + } + ], + "title": "Total", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 5 + }, + "id": 42, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "sum(kube_deployment_status_replicas_available{namespace=\"$Namespace\",deployment=~\"^$Deployment$\",pod=~\"^$Statefulset.*$\"})", + "intervalFactor": 2, + "range": true, + "refId": "A", + "step": 40 + } + ], + "title": "Available", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 5 + }, + "id": 43, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "sum(kube_deployment_status_replicas{namespace=\"$Namespace\",deployment=~\"^$Deployment$\",pod=~\"^$Statefulset.*$\"})", + "intervalFactor": 2, + "range": true, + "refId": "A", + "step": 40 + } + ], + "title": "Total", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 8 + }, + "height": "", + "hiddenSeries": false, + "id": 17, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}[1m])) by (pod)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ pod }}", + "metric": "container_cpu", + "range": true, + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Pod CPU usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:677", + "format": "none", + "label": "cores", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:678", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 15 + }, + "height": "", + "hiddenSeries": false, + "id": 24, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{pod!=\"\",pod!=\"POD\",pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}[1m])) by (pod, container)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "container: {{ pod }}:{{ container }} ", + "metric": "container_cpu", + "range": true, + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Containers CPU usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:189", + "format": "none", + "label": "cores", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:190", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 25, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}) by (pod)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ pod }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Pod memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:920", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:921", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": {}, + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 29 + }, + "hiddenSeries": false, + "id": 44, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": {}, + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"$Namespace\"}) by (pod, container)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ pod }}:{{ container }}", + "metric": "container_memory_usage:sort_desc", + "range": true, + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Container memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:920", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:921", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 16, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "editorMode": "code", + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",id!=\"\",pod=~\"^$Statefulset.*$\",namespace=\"${Namespace}\"}[2m])) by (pod)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> {{ pod }}", + "metric": "network", + "range": true, + "refId": "A", + "step": 10 + }, + { + "datasource": null, + "editorMode": "code", + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",id!=\"\",pod=~\"^$statefulset.*$\",namespace=\"${namespace}\"}[2m])) by (pod)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- {{ pod }}", + "metric": "network", + "range": true, + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Pod network I/O", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:514", + "format": "Bps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:515", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "10s", + "schemaVersion": 37, + "style": "dark", + "tags": ["kubernetes"], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "", + "value": "" + }, + "datasource": null, + "definition": "label_values(namespace)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "Namespace", + "options": [], + "query": { + "query": "label_values(namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "datasource": {}, + "definition": "label_values({namespace=\"$Namespace\"},deployment)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "Deployment", + "options": [], + "query": { + "query": "label_values({namespace=\"$Namespace\"},deployment)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "definition": "label_values({namespace=\"$Namespace\"},statefulset)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "Statefulset", + "options": [], + "query": { + "query": "label_values({namespace=\"$Namespace\"},statefulset)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": {}, + "definition": "label_values(node)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "Node", + "options": [], + "query": { + "query": "label_values(node)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "browser", + "title": "Kubernetes Deployment metrics", + "uid": "wdtm7Mc7z123", + "version": 1, + "weekStart": "" +} diff --git a/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_metrics.json b/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_metrics.json new file mode 100644 index 0000000..d3dae70 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_metrics.json @@ -0,0 +1,3390 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Summary metrics about kube-state-metrics v2 version(https://github.com/kubernetes/kube-state-metrics); Referenced 6417", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": false, + "tags": ["kubernetes-app"], + "title": "Dashboards", + "type": "dashboards" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 58, + "panels": [], + "title": "Cluster", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "#d44a3a", + "value": 90 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 4, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_info{cluster=~\"$cluster\",node=~\"$node\"}) / sum(kube_node_status_allocatable{cluster=~\"$cluster\",resource=\"pods\",node=~\"$node\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Cluster Pod Requested", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "#d44a3a", + "value": 90 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 5, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests{cluster=~\"$cluster\",resource=\"cpu\",node=~\"$node\"})/ sum(kube_node_status_allocatable{node=~\"$node\",cluster=~\"$cluster\",resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Cluster CPU Requested", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "#d44a3a", + "value": 90 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 6, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests{cluster=~\"$cluster\",resource=\"memory\",node=~\"$node\"}) / sum(kube_node_status_allocatable{node=~\"$node\",cluster=~\"$cluster\",resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Cluster Memory Requested", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "pods", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 9, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable{cluster=~\"$cluster\",resource=\"pods\",node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "allocatable", + "refId": "A" + }, + { + "expr": "sum(kube_pod_info{node=~\"$node\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "requested", + "refId": "C" + } + ], + "title": "Cluster Pod Capacity", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "cores", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 10, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_node_status_capacity{node=~\"$node\",cluster=~\"$cluster\",resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "allocatable", + "refId": "A" + }, + { + "expr": "sum(kube_node_status_allocatable{node=~\"$node\",cluster=~\"$cluster\",resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "capacity", + "refId": "B" + }, + { + "expr": "sum(kube_pod_container_resource_requests{cluster=~\"$cluster\",resource=\"cpu\",node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "requested", + "refId": "C" + }, + { + "expr": "sum(kube_pod_container_resource_limits{cluster=~\"$cluster\",resource=\"cpu\",node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "limited", + "refId": "D" + } + ], + "title": "Cluster CPU Capacity", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bits" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 11, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable{node=~\"$node\",cluster=~\"$cluster\",resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "allocatable", + "refId": "A" + }, + { + "expr": "sum(kube_node_status_capacity{node=~\"$node\",cluster=~\"$cluster\",resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "capacity", + "refId": "B" + }, + { + "expr": "sum(kube_pod_container_resource_requests{cluster=~\"$cluster\",resource=\"memory\",node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "requested", + "refId": "C" + }, + { + "expr": "sum(kube_pod_container_resource_limits{cluster=~\"$cluster\",resource=\"memory\",node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "limited", + "refId": "D" + } + ], + "title": "Cluster Mem Capacity", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 22, + "panels": [ + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 20 + }, + "id": 24, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_node_info{cluster=~\"$cluster\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number Of Nodes", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#d44a3a" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 20 + }, + "id": 26, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_node_spec_unschedulable{cluster=~\"$cluster\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Nodes Unavailable", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 78, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_node_status_condition{cluster=~\"$cluster\",condition=\"Ready\",status=\"false\"}==1", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{node}}", + "refId": "A" + } + ], + "title": "Node NotReady", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 51, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_node_status_condition{condition=\"DiskPressure\",cluster=~\"$cluster\",status=\"true\"}==1", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{node}} {{condition}}", + "refId": "A" + }, + { + "expr": "kube_node_status_condition{condition=\"MemoryPressure\",node=~\"$node\",cluster=~\"$cluster\",status=\"true\"}==1", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} {{condition}}", + "refId": "B" + }, + { + "expr": "kube_node_status_condition{condition=~\"PIDPressure\",node=~\"$node\",cluster=~\"$cluster\",status=\"true\"}==1", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{node}} {{condition}}", + "refId": "C" + } + ], + "title": "Node Pressure", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 88, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max", "min"], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_node_info{cluster=~\"$cluster\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "node number", + "type": "timeseries" + } + ], + "title": "Node", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 14, + "panels": [ + { + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "unit", + "value": "time: YYYY-MM-DD HH:mm:ss" + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Metric" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + {} + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 218 + }, + "id": 16, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_deployment_status_replicas{namespace=~\"$namespace\",cluster=~\"$cluster\"}", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ deployment }}", + "refId": "A" + } + ], + "title": "Deployment Replicas - Up To Date", + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 218 + }, + "id": 18, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_deployment_status_replicas{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Deployment Replicas", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 218 + }, + "id": 19, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_deployment_status_replicas_updated{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Deployment Replicas - Updated", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 218 + }, + "id": 20, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_deployment_status_replicas_unavailable{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Deployment Replicas - Unavailable", + "type": "stat" + } + ], + "title": "Deployments", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 71, + "panels": [ + { + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "unit", + "value": "time: YYYY-MM-DD HH:mm:ss" + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Metric" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + {} + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 22 + }, + "id": 75, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_statefulset_status_replicas_ready{namespace=~\"$namespace\",cluster=~\"$cluster\"}", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ statefulset}}", + "refId": "A" + } + ], + "title": "Statefulset Replicas - Up To Date", + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 6, + "y": 22 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_statefulset_status_replicas_ready{namespace=~\"$namespace\",cluster=~\"$cluster\"}/kube_statefulset_status_replicas{namespace=~\"$namespace\",cluster=~\"$cluster\"}*100", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{statefulset}}", + "refId": "A" + } + ], + "title": "Statefulset replicas", + "type": "timeseries" + } + ], + "title": "Statefuleset", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 28, + "panels": [ + { + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#56A64B" + }, + { + "color": "#73BF69" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "custom.align" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 23 + }, + "id": 68, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "changes(kube_pod_container_status_restarts_total{namespace=~\"$namespace\",cluster=~\"$cluster\"}[30m])>1", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Pods restart in 30m", + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 23 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_pod_status_reason{cluster=~\"$cluster\",namespace=~\"$namespace\", reason=\"Evicted\"}==1", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Evicted", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#629e51", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 30, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_status_phase{cluster=~\"$cluster\",namespace=~\"$namespace\", phase=\"Running\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Pods Running", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#629e51", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 185 + }, + "id": 33, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_status_phase{cluster=~\"$cluster\",namespace=~\"$namespace\", phase=\"Succeeded\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Pods Succeeded", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#629e51", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 185 + }, + "id": 32, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_status_phase{cluster=~\"$cluster\",namespace=~\"$namespace\", phase=\"Failed\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Pods Failed", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#629e51", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 188 + }, + "id": 31, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_status_phase{cluster=~\"$cluster\",namespace=~\"$namespace\", phase=\"Pending\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Pods Pending", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#629e51", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 188 + }, + "id": 34, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_status_phase{cluster=~\"$cluster\",namespace=~\"$namespace\", phase=\"Unknown\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Pods Unknown", + "type": "stat" + } + ], + "title": "Pods", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 36, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 250 + }, + "id": 38, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_container_status_running{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Containers Running", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 250 + }, + "id": 39, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_container_status_waiting{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Containers Waiting", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 250 + }, + "id": 40, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_container_status_terminated{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Containers Terminated", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 250 + }, + "id": 41, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(changes(kube_pod_container_status_restarts_total{namespace=~\"$namespace\",cluster=~\"$cluster\"}[30m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Containers Restarts (Last 30 Minutes)", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 0, + "y": 253 + }, + "id": 43, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "CPU Cores Requested by Containers", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 12, + "y": 253 + }, + "id": 42, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~\"$namespace\" ,cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Memory Requested By Containers", + "type": "stat" + } + ], + "title": "Containers", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 80, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 82, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_hpa_status_current_replicas{cluster=~\"$cluster\",namespace=~\"$namespace\"}", + "instant": false, + "interval": "", + "legendFormat": "current_{{hpa}}", + "refId": "A" + }, + { + "expr": "kube_hpa_spec_max_replicas{cluster=~\"$cluster\",namespace=~\"$namespace\"}", + "instant": false, + "interval": "", + "legendFormat": "max_{{hpa}}", + "refId": "B" + }, + { + "expr": "kube_hpa_spec_min_replicas{cluster=~\"$cluster\",namespace=~\"$namespace\"}", + "instant": false, + "interval": "", + "legendFormat": "min_{{hpa}}", + "refId": "C" + } + ], + "title": "hpa", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 108 + }, + "id": 89, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_hpa_status_current_replicas{hpa=~\".*\"})", + "instant": false, + "interval": "", + "legendFormat": "kube_hpa_status_current_replicas", + "refId": "A" + } + ], + "title": "total-hpa-current", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 116 + }, + "id": 90, + "options": {}, + "pluginVersion": "7.1.1", + "targets": [ + { + "expr": "kube_hpa_status_current_replicas{hpa=~\".*\"} == kube_hpa_spec_max_replicas{hpa=~\".*\"}", + "instant": false, + "interval": "", + "legendFormat": "{{hpa}}", + "refId": "A" + } + ], + "title": "current==max", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 116 + }, + "id": 92, + "options": {}, + "pluginVersion": "7.1.1", + "targets": [ + { + "expr": "kube_hpa_status_current_replicas{hpa=~\".*\"} == kube_hpa_spec_min_replicas{hpa=~\".*\"}", + "instant": false, + "interval": "", + "legendFormat": "{{hpa}}", + "refId": "B" + } + ], + "title": "current==min", + "type": "timeseries" + } + ], + "title": "HPA", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 45, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 26 + }, + "id": 47, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_job_status_succeeded{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Jobs Succeeded", + "type": "stat" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 26 + }, + "id": 62, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "(time()-kube_job_status_completion_time{namespace=~\"$namespace\",cluster=~\"$cluster\"})/60/60/24>1", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{job_name}}", + "refId": "A" + } + ], + "title": "Job last success time from now", + "transparent": true, + "type": "table" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 26 + }, + "id": 76, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_job_status_failed{namespace=~\"$namespace\",cluster=~\"$cluster\"}>1", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{job_name}}", + "refId": "A" + } + ], + "title": "Job failed", + "transparent": true, + "type": "table" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 37 + }, + "id": 48, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_job_status_active{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Jobs Active", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 40 + }, + "id": 49, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_job_status_failed{namespace=~\"$namespace\",cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Jobs Failed", + "type": "stat" + } + ], + "title": "Jobs", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 64, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 27 + }, + "id": 66, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_persistentvolumeclaim_status_phase{phase=\"Bound\"}==1)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{persistentvolumeclaim=}}", + "refId": "A" + } + ], + "title": "pvc is bound", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 27 + }, + "id": 69, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "sum(kube_persistentvolumeclaim_status_phase{phase!=\"Bound\"}==1)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{persistentvolumeclaim=}}", + "refId": "A" + } + ], + "title": "pvc is not bound", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 67, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "expr": "kube_persistentvolumeclaim_resource_requests_storage_bytes/1024/1024/1024", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{persistentvolumeclaim}}", + "refId": "A" + } + ], + "title": "pvc", + "type": "table" + } + ], + "title": "PVC", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": ["kubernetes", "kubernetes-app"], + "templating": { + "list": [ + { + "current": { + "text": "", + "value": "" + }, + "definition": "label_values(kube_node_info, cluster)", + "includeAll": false, + "name": "cluster", + "options": [], + "query": "label_values(kube_node_info, cluster)", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(kube_node_info{cluster=~\"$cluster\"}, node)", + "includeAll": true, + "name": "node", + "options": [], + "query": "label_values(kube_node_info{cluster=~\"$cluster\"}, node)", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "", + "includeAll": true, + "name": "namespace", + "options": [], + "query": "label_values(kube_namespace_created{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 2, + "name": "datasource", + "query": "Prometheus", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "browser", + "title": "kube-state-metrics-v2", + "uid": "garysdevil-kube-state-metrics-v2", + "version": 1 +} diff --git a/rootfs/usr/share/grafana/api/dashboards/main/node.json b/rootfs/usr/share/grafana/api/dashboards/main/node.json new file mode 100644 index 0000000..e4e067d --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/main/node.json @@ -0,0 +1,3613 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "cpu memroy disk IO network temperature", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 8919, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "More", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 176, + "panels": [], + "title": "Basic info", + "type": "row" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "editable": true, + "error": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 11, + "links": [], + "maxPerRow": 6, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "", + "mode": "html" + }, + "pluginVersion": "9.5.3", + "repeat": "node", + "repeatDirection": "h", + "style": {}, + "title": "$node", + "type": "text" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 2 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 0, + "y": 2 + }, + "hideTimeOverride": true, + "id": 15, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "time() - node_boot_time_seconds{instance=~\"$node\"}", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 2, + "refId": "A", + "step": 40 + } + ], + "title": "uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 2 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 2, + "y": 2 + }, + "id": 14, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "expr": "count(count(node_cpu_seconds_total{instance=~\"$node\", mode='system'}) by (cpu))", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 20 + } + ], + "title": "cpu cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 2 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 3 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 4, + "y": 2 + }, + "id": 75, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "title": "memory", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 6, + "y": 2 + }, + "hideTimeOverride": false, + "id": 171, + "links": [], + "maxDataPoints": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum(node_filesystem_size_bytes {instance=~\"$node\",fstype=~\"ext4|xfs\"})", + "format": "table", + "hide": false, + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "refId": "A" + } + ], + "title": "system disk", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 50 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 8, + "y": 2 + }, + "id": 167, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "expr": "100 - (avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"idle\"}[5m])) * 100)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 20 + } + ], + "title": "cpu usage（5m）", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 20 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 50 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 11, + "y": 2 + }, + "id": 20, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"iowait\"}[5m])) * 100", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 20 + } + ], + "title": "CPU iowait（5m）", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 14, + "y": 2 + }, + "hideTimeOverride": false, + "id": 172, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "expr": "(1 - (node_memory_MemAvailable_bytes{instance=~\"$node\"} / (node_memory_MemTotal_bytes{instance=~\"$node\"})))* 100", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 20 + } + ], + "title": "Memory Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "Obtain the maximum partition through the variable maxmount.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 17, + "y": 2 + }, + "id": 154, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.5.3", + "repeatDirection": "h", + "targets": [ + { + "expr": "100 - ((node_filesystem_avail_bytes{instance=~\"$node\",mountpoint=\"$maxmount\",fstype=~\"ext4|xfs\"} * 100) / node_filesystem_size_bytes {instance=~\"$node\",mountpoint=\"$maxmount\",fstype=~\"ext4|xfs\"})", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 20 + } + ], + "title": "partion ($maxmount) usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "max": 10000, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 7000 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 9000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 21, + "y": 2 + }, + "hideTimeOverride": false, + "id": 16, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "expr": "node_filefd_allocated{instance=~\"$node\"}", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "refId": "B" + } + ], + "title": "opend file descriptor", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "15 minutes" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "1 minute" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5分钟" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 0, + "y": 7 + }, + "id": 13, + "links": [], + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "expr": "node_load1{instance=~\"$node\"}", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 2, + "legendFormat": "{{instance}}_1m", + "metric": "", + "refId": "A", + "step": 20, + "target": "" + }, + { + "expr": "node_load5{instance=~\"$node\"}", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 2, + "legendFormat": "{{instance}}_5m", + "refId": "B", + "step": 20 + }, + { + "expr": "node_load15{instance=~\"$node\"}", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 2, + "legendFormat": "{{instance}}_15m", + "refId": "C", + "step": 20 + } + ], + "title": "Load average", + "type": "timeseries" + }, + { + "aliasColors": { + "Idle - Waiting for something to happen": "#052B51", + "guest": "#9AC48A", + "idle": "#052B51", + "iowait": "#EAB839", + "irq": "#BF1B00", + "nice": "#C15C17", + "sdb_I/O operations per second%": "#d683ce", + "softirq": "#E24D42", + "steal": "#FCE2DE", + "system": "#508642", + "user": "#5195CE", + "The proportion of disk spent on I/O operations": "#ba43a9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "decimals": 2, + "description": "node_disk_io_time_seconds_total", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 13, + "x": 11, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"system\"}[2m])) by (instance)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}_System", + "refId": "A", + "step": 20 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"user\"}[2m])) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}_User", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"idle\"}[2m])) by (instance)", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{instance}}_Idle", + "range": true, + "refId": "F", + "step": 240 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"iowait\"}[2m])) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}_Iowait", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}_{{device}}_per second I/O%", + "range": true, + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU usage、disk I/O time（%）", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:513", + "format": "percentunit", + "label": "", + "logBase": 1, + "max": "1", + "show": true + }, + { + "$$hashKey": "object:514", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "displayName": "", + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 11, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 207, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"}-node_filesystem_avail_bytes {instance=~'$node',fstype=~\"ext4|xfs\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}_usage", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Partion usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3570", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:3571", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "mountpoint" + }, + "properties": [ + { + "id": "displayName", + "value": "partion" + }, + { + "id": "unit", + "value": "bytes" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)" + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "free" + }, + { + "id": "unit", + "value": "bytes" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10000000000 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 20000000000 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "usage" + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "displayName", + "value": "total" + }, + { + "id": "unit", + "value": "bytes" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "fstype" + }, + "properties": [ + { + "id": "displayName", + "value": "file system" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "displayName", + "value": "file system" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "device" + }, + "properties": [ + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 13, + "x": 11, + "y": 14 + }, + "id": 164, + "links": [], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "file system" + } + ] + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "exemplar": false, + "expr": "node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "C" + }, + { + "exemplar": true, + "expr": "node_filesystem_avail_bytes {instance=~'$node',fstype=~\"ext4|xfs\"}", + "format": "table", + "hide": true, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + }, + { + "exemplar": false, + "expr": "1-(node_filesystem_free_bytes{instance=~'$node',fstype=~\"ext4|xfs\"} / node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "B" + } + ], + "title": "Disk usage", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "fstype", + "mountpoint", + "Value #C", + "Value #A", + "Value #B", + "device" + ] + } + } + } + ], + "type": "table" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 211, + "panels": [ + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "MEM_Avaliable" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "MEM_Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "MEM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "MEM_Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6d1f62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "MEM_Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#eab839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "avaliable" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9ac48a", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "totol memery" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#bf1b00", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 2 + }, + "id": 156, + "links": [], + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}_TotalMem", + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"} - node_memory_MemAvailable_bytes{instance=~\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}_Used", + "range": true, + "refId": "B", + "step": 4 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_MemFree_bytes{instance=~\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "MEM_Free", + "range": true, + "refId": "C", + "step": 4 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_Buffers_bytes{instance=~\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "MEM_Buffers", + "range": true, + "refId": "D", + "step": 4 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_Cached_bytes{instance=~\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "MEM_Cached", + "range": true, + "refId": "E", + "step": 4 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_MemAvailable_bytes{instance=~\"$node\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}_Available", + "range": true, + "refId": "F", + "step": 4 + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"} - (node_memory_Cached_bytes{instance=~\"$node\"} + node_memory_Buffers_bytes{instance=~\"$node\"} + node_memory_MemFree_bytes{instance=~\"$node\"})", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "refId": "G" + } + ], + "title": "Memory info", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 2 + }, + "id": 206, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "exemplar": true, + "expr": "node_memory_SwapFree_bytes{instance=~\"$node\"}", + "hide": false, + "interval": "", + "legendFormat": "Swap_Free", + "refId": "A" + }, + { + "exemplar": true, + "expr": "node_memory_SwapCached_bytes{instance=~\"$node\"}", + "hide": false, + "interval": "", + "legendFormat": "SwapCached", + "refId": "B" + }, + { + "exemplar": true, + "expr": "node_memory_SwapTotal_bytes{instance=~\"$node\"}", + "hide": false, + "interval": "", + "legendFormat": "SwapTotal", + "refId": "C" + } + ], + "title": "Swap memory", + "type": "timeseries" + } + ], + "title": "Memory", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 209, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 3 + }, + "height": "300", + "hiddenSeries": false, + "id": 157, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:101", + "alias": "/.*_out transmit$/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_network_receive_bytes_total{instance=~'$node',device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*|lxc.*|nodelocaldns|cilium.*'}[5m])*8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}_{{device}}_in receive", + "range": true, + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_network_transmit_bytes_total{instance=~'$node',device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*|lxc.*|nodelocaldns|cilium.*'}[5m])*8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}_{{device}}_out transmit", + "range": true, + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network flow", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:85", + "format": "bps", + "label": "upload（-）/download（+）", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:86", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "TCP": "#6ED0E0" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 3 + }, + "height": "300", + "hiddenSeries": false, + "id": 158, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_netstat_Tcp_CurrEstab{instance=~'$node'}", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{instance}}_ESTABLISHED", + "refId": "A", + "step": 20 + }, + { + "expr": "node_sockstat_TCP_tw{instance=~'$node'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}_TCP_tw", + "refId": "D" + }, + { + "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=~'$node'}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}_ActiveOpens", + "refId": "B" + }, + { + "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=~'$node'}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}_PassiveOpens", + "refId": "C" + }, + { + "expr": "node_sockstat_TCP_alloc{instance=~'$node'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}_TCP_alloc", + "refId": "E" + }, + { + "expr": "node_sockstat_TCP_inuse{instance=~'$node'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}_TCP_inuse", + "refId": "F" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TCP connect info", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:269", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:270", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Network", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 190, + "panels": [ + { + "aliasColors": { + "vda_write": "#6ED0E0" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "Reads completed: Reads completed per second for each disk partition\n\nWrites completed: \nIO now", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 24 + }, + "height": "300", + "hiddenSeries": false, + "id": 161, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:386", + "alias": "/.*_read$/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_reads_completed_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}_{{device}}_read", + "range": true, + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_writes_completed_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}_{{device}}_write", + "range": true, + "refId": "B", + "step": 10 + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "node_disk_io_now{instance=~\"$node\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "disk read and write（IOPS）", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:393", + "format": "iops", + "label": "read（-）/write（+）I/O ops/sec", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:394", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "vda": "#6ED0E0" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "Read time ms The number of seconds taken for each disk partition read operation\n\nWrite time ms\nIO time ms \nIO time weighted", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 24 + }, + "height": "300", + "hiddenSeries": false, + "id": 160, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:459", + "alias": "/,*_read$/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "range": true, + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}}_weighting", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_read_time_seconds_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}_{{device}}_read", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_write_time_seconds_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}_{{device}}_write", + "range": true, + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "disk io read write time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:466", + "format": "s", + "label": "read（-）/write（+）", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:467", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "The number of read operations per second=[Δrd_ios/Δt]", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 194, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideZero": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_disk_reads_completed_total{instance=~'$node'}[2m])", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "r/s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1217", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1218", + "format": "bytes", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "The number of write operations per second=[Δwr_ios/Δt]", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "hiddenSeries": false, + "id": 196, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "(rate(node_disk_writes_completed_total{instance=~\"$node\"}[2m]))", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "w/s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1363", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1364", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "The average time required for each read operation=[ Δ Rd_ticks/ Δ Rd_ios not only includes the time required for hard disk device read operations, but also the waiting time in the kernel queue.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "hiddenSeries": false, + "id": 202, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "(rate(node_disk_read_time_seconds_total {instance=~'$node'}[2m])) /(rate(node_disk_reads_completed_total {instance=~'$node'}[2m]))", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "r_await", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2148", + "format": "s", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2149", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "Average time required for each write operation=[ Δ Wr_ticks/ Δ Wr_ios not only includes the time required for hard disk device write operations, but also the waiting time in the kernel queue.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "hiddenSeries": false, + "id": 204, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "(rate(node_disk_write_time_seconds_total {instance=~'$node'}[2m])) /(rate(node_disk_writes_completed_total {instance=~'$node'}[2m]))", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "w_await", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2477", + "format": "s", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2478", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 198, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "(rate(node_disk_read_bytes_total {instance=~'$node'}[2m]))", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "rMiB/s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1622", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1623", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "hiddenSeries": false, + "id": 200, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "(rate(node_disk_written_bytes_total {instance=~'$node'}[2m]))", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "wKiB/s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1920", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1921", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "vda_write": "#6ED0E0" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 55 + }, + "height": "300", + "hiddenSeries": false, + "id": 168, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*_read$/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_read_bytes_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}_{{device}}_read", + "range": true, + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_written_bytes_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}_{{device}}_write", + "range": true, + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk read and write capacity size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": "read（-）/write（+）", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "The busy ratio of this hard drive device=[ Δ Io_ticks/ Δ t] The time ratio indicating that the device has I/O (i.e. non idle), regardless of how much I/O there is, only considering whether there is", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 65 + }, + "hiddenSeries": false, + "id": 192, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideZero": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "idc", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[2m])", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "%util", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:744", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:745", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Disk read write", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 213, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 169, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "node_hwmon_temp_celsius{instance=~'$node'}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{chip}}_{{sensor}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cpu temperature", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3976", + "format": "celsius", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:3977", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Cpu temperature", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": ["instance"], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 10, + "auto_min": "10s", + "current": { + "selected": true, + "text": "5s", + "value": "5s" + }, + "hide": 0, + "label": "interval", + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": true, + "text": "5s", + "value": "5s" + }, + { + "selected": false, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "allFormat": "glob", + "allValue": "", + "current": { + "selected": false, + "text": [], + "value": [] + }, + "datasource": { + "type": "prometheus" + }, + "definition": "label_values(node_exporter_build_info{},instance)", + "hide": 0, + "includeAll": false, + "label": "host instance", + "multi": true, + "multiFormat": "regex values", + "name": "node", + "options": [], + "query": { + "query": "label_values(node_exporter_build_info{},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "/", + "value": "/" + }, + "datasource": { + "type": "prometheus" + }, + "definition": "", + "hide": 2, + "includeAll": false, + "label": "", + "multi": false, + "name": "maxmount", + "options": [], + "query": { + "query": "query_result(topk(1,sort_desc (max(node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"}) by (mountpoint))))", + "refId": "Prometheus-maxmount-Variable-Query" + }, + "refresh": 2, + "regex": "/.*\\\"(.*)\\\".*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "datasource": {}, + "filters": [], + "hide": 0, + "label": "", + "name": "Filters", + "skipUrlSync": false, + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "browser", + "title": "Node Metrics", + "uid": "SQlD2vTZq", + "version": 1, + "weekStart": "" +} diff --git a/rootfs/usr/share/grafana/api/dashboards/node.json b/rootfs/usr/share/grafana/api/dashboards/node.json deleted file mode 100644 index 0eabd6b..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/node.json +++ /dev/null @@ -1,3657 +0,0 @@ -{ - "dashboard": { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "cpu memroy disk IO network temperature", - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 8919, - "graphTooltip": 0, - "id": null, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [], - "targetBlank": true, - "title": "More", - "tooltip": "", - "type": "dashboards", - "url": "" - } - ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "prometheus" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 176, - "panels": [], - "title": "Basic info", - "type": "row" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "editable": true, - "error": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 11, - "links": [], - "maxPerRow": 6, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "", - "mode": "html" - }, - "pluginVersion": "9.5.3", - "repeat": "node", - "repeatDirection": "h", - "style": {}, - "title": "$node", - "type": "text" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 1 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 2 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 2, - "x": 0, - "y": 2 - }, - "hideTimeOverride": true, - "id": 15, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "time() - node_boot_time_seconds{instance=~\"$node\"}", - "format": "time_series", - "hide": false, - "instant": true, - "intervalFactor": 2, - "refId": "A", - "step": 40 - } - ], - "title": "uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 1 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 2 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 2, - "x": 2, - "y": 2 - }, - "id": 14, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "expr": "count(count(node_cpu_seconds_total{instance=~\"$node\", mode='system'}) by (cpu))", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 20 - } - ], - "title": "cpu cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 2 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 3 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 2, - "x": 4, - "y": 2 - }, - "id": 75, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 20 - } - ], - "title": "memory", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 2, - "x": 6, - "y": 2 - }, - "hideTimeOverride": false, - "id": 171, - "links": [], - "maxDataPoints": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "value" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "exemplar": true, - "expr": "sum(node_filesystem_size_bytes {instance=~\"$node\",fstype=~\"ext4|xfs\"})", - "format": "table", - "hide": false, - "instant": false, - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{mountpoint}}", - "refId": "A" - } - ], - "title": "system disk", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 50 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 8, - "y": 2 - }, - "id": 167, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "expr": "100 - (avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"idle\"}[5m])) * 100)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 20 - } - ], - "title": "cpu usage（5m）", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 20 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 50 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 11, - "y": 2 - }, - "id": 20, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"iowait\"}[5m])) * 100", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 20 - } - ], - "title": "CPU iowait（5m）", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 80 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 14, - "y": 2 - }, - "hideTimeOverride": false, - "id": 172, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "expr": "(1 - (node_memory_MemAvailable_bytes{instance=~\"$node\"} / (node_memory_MemTotal_bytes{instance=~\"$node\"})))* 100", - "format": "time_series", - "hide": false, - "interval": "10s", - "intervalFactor": 1, - "refId": "A", - "step": 20 - } - ], - "title": "Memory Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "Obtain the maximum partition through the variable maxmount.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 70 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 17, - "y": 2 - }, - "id": 154, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "9.5.3", - "repeatDirection": "h", - "targets": [ - { - "expr": "100 - ((node_filesystem_avail_bytes{instance=~\"$node\",mountpoint=\"$maxmount\",fstype=~\"ext4|xfs\"} * 100) / node_filesystem_size_bytes {instance=~\"$node\",mountpoint=\"$maxmount\",fstype=~\"ext4|xfs\"})", - "format": "time_series", - "interval": "10s", - "intervalFactor": 1, - "refId": "A", - "step": 20 - } - ], - "title": "partion ($maxmount) usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [], - "max": 10000, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 7000 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 9000 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 21, - "y": 2 - }, - "hideTimeOverride": false, - "id": 16, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "expr": "node_filefd_allocated{instance=~\"$node\"}", - "format": "time_series", - "instant": false, - "interval": "10s", - "intervalFactor": 1, - "refId": "B" - } - ], - "title": "opend file descriptor", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "15 minutes" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "1 minute" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "5分钟" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CCA300", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 11, - "x": 0, - "y": 7 - }, - "id": 13, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.2.3", - "targets": [ - { - "expr": "node_load1{instance=~\"$node\"}", - "format": "time_series", - "instant": false, - "interval": "10s", - "intervalFactor": 2, - "legendFormat": "{{instance}}_1m", - "metric": "", - "refId": "A", - "step": 20, - "target": "" - }, - { - "expr": "node_load5{instance=~\"$node\"}", - "format": "time_series", - "instant": false, - "interval": "10s", - "intervalFactor": 2, - "legendFormat": "{{instance}}_5m", - "refId": "B", - "step": 20 - }, - { - "expr": "node_load15{instance=~\"$node\"}", - "format": "time_series", - "instant": false, - "interval": "10s", - "intervalFactor": 2, - "legendFormat": "{{instance}}_15m", - "refId": "C", - "step": 20 - } - ], - "title": "Load average", - "type": "timeseries" - }, - { - "aliasColors": { - "Idle - Waiting for something to happen": "#052B51", - "guest": "#9AC48A", - "idle": "#052B51", - "iowait": "#EAB839", - "irq": "#BF1B00", - "nice": "#C15C17", - "sdb_I/O operations per second%": "#d683ce", - "softirq": "#E24D42", - "steal": "#FCE2DE", - "system": "#508642", - "user": "#5195CE", - "The proportion of disk spent on I/O operations": "#ba43a9" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "decimals": 2, - "description": "node_disk_io_time_seconds_total", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 13, - "x": 11, - "y": 7 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxPerRow": 6, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"system\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}_System", - "refId": "A", - "step": 20 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"user\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}_User", - "range": true, - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"idle\"}[2m])) by (instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{instance}}_Idle", - "range": true, - "refId": "F", - "step": 240 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"iowait\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}_Iowait", - "range": true, - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}_{{device}}_per second I/O%", - "range": true, - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU usage、disk I/O time（%）", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:513", - "format": "percentunit", - "label": "", - "logBase": 1, - "max": "1", - "show": true - }, - { - "$$hashKey": "object:514", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "displayName": "", - "unit": "bytes" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 11, - "x": 0, - "y": 14 - }, - "hiddenSeries": false, - "id": 207, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"}-node_filesystem_avail_bytes {instance=~'$node',fstype=~\"ext4|xfs\"}", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{mountpoint}}_usage", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Partion usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transformations": [], - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3570", - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:3571", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": false, - "inspect": false - }, - "decimals": 2, - "displayName": "", - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "mountpoint" - }, - "properties": [ - { - "id": "displayName", - "value": "partion" - }, - { - "id": "unit", - "value": "bytes" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)" - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #A" - }, - "properties": [ - { - "id": "displayName", - "value": "free" - }, - { - "id": "unit", - "value": "bytes" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 10000000000 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 20000000000 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #B" - }, - "properties": [ - { - "id": "displayName", - "value": "usage" - }, - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "color-background" - } - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 70 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #C" - }, - "properties": [ - { - "id": "displayName", - "value": "total" - }, - { - "id": "unit", - "value": "bytes" - }, - { - "id": "decimals", - "value": 1 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "fstype" - }, - "properties": [ - { - "id": "displayName", - "value": "file system" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "displayName", - "value": "file system" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "device" - }, - "properties": [ - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 13, - "x": 11, - "y": 14 - }, - "id": 164, - "links": [], - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "file system" - } - ] - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "exemplar": false, - "expr": "node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "C" - }, - { - "exemplar": true, - "expr": "node_filesystem_avail_bytes {instance=~'$node',fstype=~\"ext4|xfs\"}", - "format": "table", - "hide": true, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - }, - { - "exemplar": false, - "expr": "1-(node_filesystem_free_bytes{instance=~'$node',fstype=~\"ext4|xfs\"} / node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "B" - } - ], - "title": "Disk usage", - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "fstype", - "mountpoint", - "Value #C", - "Value #A", - "Value #B", - "device" - ] - } - } - } - ], - "type": "table" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 211, - "panels": [ - { - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "MEM_Avaliable" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "MEM_Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "MEM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "MEM_Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6d1f62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "MEM_Used" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#eab839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "avaliable" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9ac48a", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "totol memery" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#bf1b00", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 2 - }, - "id": 156, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"}", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}_TotalMem", - "refId": "A", - "step": 4 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"} - node_memory_MemAvailable_bytes{instance=~\"$node\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}_Used", - "range": true, - "refId": "B", - "step": 4 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_MemFree_bytes{instance=~\"$node\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "MEM_Free", - "range": true, - "refId": "C", - "step": 4 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_Buffers_bytes{instance=~\"$node\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "MEM_Buffers", - "range": true, - "refId": "D", - "step": 4 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_Cached_bytes{instance=~\"$node\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "MEM_Cached", - "range": true, - "refId": "E", - "step": 4 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_MemAvailable_bytes{instance=~\"$node\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}_Available", - "range": true, - "refId": "F", - "step": 4 - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"} - (node_memory_Cached_bytes{instance=~\"$node\"} + node_memory_Buffers_bytes{instance=~\"$node\"} + node_memory_MemFree_bytes{instance=~\"$node\"})", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "refId": "G" - } - ], - "title": "Memory info", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 2 - }, - "id": 206, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.2.3", - "targets": [ - { - "exemplar": true, - "expr": "node_memory_SwapFree_bytes{instance=~\"$node\"}", - "hide": false, - "interval": "", - "legendFormat": "Swap_Free", - "refId": "A" - }, - { - "exemplar": true, - "expr": "node_memory_SwapCached_bytes{instance=~\"$node\"}", - "hide": false, - "interval": "", - "legendFormat": "SwapCached", - "refId": "B" - }, - { - "exemplar": true, - "expr": "node_memory_SwapTotal_bytes{instance=~\"$node\"}", - "hide": false, - "interval": "", - "legendFormat": "SwapTotal", - "refId": "C" - } - ], - "title": "Swap memory", - "type": "timeseries" - } - ], - "title": "Memory", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 209, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 3 - }, - "height": "300", - "hiddenSeries": false, - "id": 157, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:101", - "alias": "/.*_out transmit$/", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_network_receive_bytes_total{instance=~'$node',device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*|lxc.*|nodelocaldns|cilium.*'}[5m])*8", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}_{{device}}_in receive", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_network_transmit_bytes_total{instance=~'$node',device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*|lxc.*|nodelocaldns|cilium.*'}[5m])*8", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}_{{device}}_out transmit", - "range": true, - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Network flow", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:85", - "format": "bps", - "label": "upload（-）/download（+）", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:86", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "TCP": "#6ED0E0" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 3 - }, - "height": "300", - "hiddenSeries": false, - "id": 158, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node_netstat_Tcp_CurrEstab{instance=~'$node'}", - "format": "time_series", - "hide": false, - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{instance}}_ESTABLISHED", - "refId": "A", - "step": 20 - }, - { - "expr": "node_sockstat_TCP_tw{instance=~'$node'}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}_TCP_tw", - "refId": "D" - }, - { - "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=~'$node'}[1m])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}_ActiveOpens", - "refId": "B" - }, - { - "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=~'$node'}[1m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}_PassiveOpens", - "refId": "C" - }, - { - "expr": "node_sockstat_TCP_alloc{instance=~'$node'}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}_TCP_alloc", - "refId": "E" - }, - { - "expr": "node_sockstat_TCP_inuse{instance=~'$node'}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}_TCP_inuse", - "refId": "F" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TCP connect info", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:269", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:270", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "title": "Network", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 190, - "panels": [ - { - "aliasColors": { - "vda_write": "#6ED0E0" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "Reads completed: Reads completed per second for each disk partition\n\nWrites completed: \nIO now", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 24 - }, - "height": "300", - "hiddenSeries": false, - "id": 161, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:386", - "alias": "/.*_read$/", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_reads_completed_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}_{{device}}_read", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_writes_completed_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}_{{device}}_write", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "node_disk_io_now{instance=~\"$node\"}", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{device}}", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "disk read and write（IOPS）", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:393", - "format": "iops", - "label": "read（-）/write（+）I/O ops/sec", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:394", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "vda": "#6ED0E0" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "Read time ms The number of seconds taken for each disk partition read operation\n\nWrite time ms\nIO time ms \nIO time weighted", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 3, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 24 - }, - "height": "300", - "hiddenSeries": false, - "id": 160, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:459", - "alias": "/,*_read$/", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{device}}", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{device}}_weighting", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_read_time_seconds_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}_{{device}}_read", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_write_time_seconds_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}_{{device}}_write", - "range": true, - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "disk io read write time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:466", - "format": "s", - "label": "read（-）/write（+）", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:467", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "The number of read operations per second=[Δrd_ios/Δt]", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 31 - }, - "hiddenSeries": false, - "id": 194, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "rate(node_disk_reads_completed_total{instance=~'$node'}[2m])", - "interval": "", - "legendFormat": "{{device}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "r/s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1217", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1218", - "format": "bytes", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "The number of write operations per second=[Δwr_ios/Δt]", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 31 - }, - "hiddenSeries": false, - "id": 196, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "(rate(node_disk_writes_completed_total{instance=~\"$node\"}[2m]))", - "interval": "", - "legendFormat": "{{device}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "w/s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1363", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1364", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "The average time required for each read operation=[ Δ Rd_ticks/ Δ Rd_ios not only includes the time required for hard disk device read operations, but also the waiting time in the kernel queue.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 39 - }, - "hiddenSeries": false, - "id": 202, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "(rate(node_disk_read_time_seconds_total {instance=~'$node'}[2m])) /(rate(node_disk_reads_completed_total {instance=~'$node'}[2m]))", - "interval": "", - "legendFormat": "{{device}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "r_await", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:2148", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:2149", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "Average time required for each write operation=[ Δ Wr_ticks/ Δ Wr_ios not only includes the time required for hard disk device write operations, but also the waiting time in the kernel queue.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 39 - }, - "hiddenSeries": false, - "id": 204, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "(rate(node_disk_write_time_seconds_total {instance=~'$node'}[2m])) /(rate(node_disk_writes_completed_total {instance=~'$node'}[2m]))", - "interval": "", - "legendFormat": "{{device}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "w_await", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:2477", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:2478", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 47 - }, - "hiddenSeries": false, - "id": 198, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "(rate(node_disk_read_bytes_total {instance=~'$node'}[2m]))", - "interval": "", - "legendFormat": "{{device}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "rMiB/s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1622", - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1623", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 47 - }, - "hiddenSeries": false, - "id": 200, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "(rate(node_disk_written_bytes_total {instance=~'$node'}[2m]))", - "interval": "", - "legendFormat": "{{device}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "wKiB/s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1920", - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1921", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "vda_write": "#6ED0E0" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 55 - }, - "height": "300", - "hiddenSeries": false, - "id": 168, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*_read$/", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_read_bytes_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}_{{device}}_read", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_written_bytes_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}_{{device}}_write", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Disk read and write capacity size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": "read（-）/write（+）", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "The busy ratio of this hard drive device=[ Δ Io_ticks/ Δ t] The time ratio indicating that the device has I/O (i.e. non idle), regardless of how much I/O there is, only considering whether there is", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 65 - }, - "hiddenSeries": false, - "id": 192, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeat": "idc", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[2m])", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{device}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "%util", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:744", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:745", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "title": "Disk read write", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 213, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 25 - }, - "hiddenSeries": false, - "id": 169, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 0.5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "node_hwmon_temp_celsius{instance=~'$node'}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{chip}}_{{sensor}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "cpu temperature", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3976", - "format": "celsius", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:3977", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "title": "Cpu temperature", - "type": "row" - } - ], - "refresh": "", - "schemaVersion": 38, - "style": "dark", - "tags": [ - "instance" - ], - "templating": { - "list": [ - { - "auto": true, - "auto_count": 10, - "auto_min": "10s", - "current": { - "selected": true, - "text": "5s", - "value": "5s" - }, - "hide": 0, - "label": "interval", - "name": "interval", - "options": [ - { - "selected": false, - "text": "auto", - "value": "$__auto_interval_interval" - }, - { - "selected": true, - "text": "5s", - "value": "5s" - }, - { - "selected": false, - "text": "10s", - "value": "10s" - }, - { - "selected": false, - "text": "30s", - "value": "30s" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - } - ], - "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - }, - { - "allFormat": "glob", - "allValue": "", - "current": { - "selected": false, - "text": [ - ], - "value": [ - ] - }, - "datasource": { - "type": "prometheus" - }, - "definition": "label_values(node_exporter_build_info{},instance)", - "hide": 0, - "includeAll": false, - "label": "host instance", - "multi": true, - "multiFormat": "regex values", - "name": "node", - "options": [], - "query": { - "query": "label_values(node_exporter_build_info{},instance)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "/", - "value": "/" - }, - "datasource": { - "type": "prometheus" - }, - "definition": "", - "hide": 2, - "includeAll": false, - "label": "", - "multi": false, - "name": "maxmount", - "options": [], - "query": { - "query": "query_result(topk(1,sort_desc (max(node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"}) by (mountpoint))))", - "refId": "Prometheus-maxmount-Variable-Query" - }, - "refresh": 2, - "regex": "/.*\\\"(.*)\\\".*/", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "datasource": { - }, - "filters": [], - "hide": 0, - "label": "", - "name": "Filters", - "skipUrlSync": false, - "type": "adhoc" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Node Metrics", - "uid": "SQlD2vTZq", - "version": 1, - "weekStart": "" - }, - "overwrite": false -} \ No newline at end of file diff --git a/rootfs/usr/share/grafana/api/dashboards/user/user_container.json b/rootfs/usr/share/grafana/api/dashboards/user/user_container.json new file mode 100644 index 0000000..daa1104 --- /dev/null +++ b/rootfs/usr/share/grafana/api/dashboards/user/user_container.json @@ -0,0 +1,2770 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 33, + "panels": [], + "targets": [ + { + "refId": "A" + } + ], + "title": "SUM", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 31, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_namespace", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",container!=\"\",namespace=\"drycc\"}[1m])) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["namespace"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total CPU Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 32, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_namespace", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",container!=\"\",namespace=\"drycc\"})", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["namespace"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 52, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 2 + }, + "hiddenSeries": false, + "id": 54, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",container=~\"^drycc-passport.*$\"}[1m])) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 2 + }, + "hiddenSeries": false, + "id": 56, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-passport.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "PASSPORT", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 34, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "hiddenSeries": false, + "id": 1, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-controller.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "hiddenSeries": false, + "id": 2, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-controller.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "CONTROLLER", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 35, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 4 + }, + "hiddenSeries": false, + "id": 3, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-builder.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 4 + }, + "hiddenSeries": false, + "id": 4, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-builder.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "BUILDER", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 36, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 6, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-database.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 12 + }, + "hiddenSeries": false, + "id": 5, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-database.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "DATABASE", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 37, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 13 + }, + "hiddenSeries": false, + "id": 19, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-logger-fluentbit.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 13 + }, + "hiddenSeries": false, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-logger-fluentbit.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "FLUENTBIT", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 38, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 25, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "datasource": null, + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-grafana.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 26, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "datasource": null, + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-grafana.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "GRAFANA", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 40, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 7, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=\"drycc-logger\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 8, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=\"drycc-logger\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "LOGGER", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 41, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 9, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 17 + }, + "hiddenSeries": false, + "id": 10, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "STORAGE", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 43, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 27, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-valkey.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 28, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-valkey.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "VALKEY", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 44, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 11, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-registry.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 22 + }, + "hiddenSeries": false, + "id": 12, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=\"drycc-registry\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "REGISTRY", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 45, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 16, + "interval": "", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-registry-proxy.*$\"}[1m])) by (pod) * 1000", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Millicores", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 23 + }, + "hiddenSeries": false, + "id": 15, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_pod_name", + "editorMode": "code", + "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-registry-proxy.*$\"}) by (pod)", + "groupBy": [ + { + "params": ["$interval"], + "type": "time" + }, + { + "params": ["pod_name"], + "type": "tag" + }, + { + "params": ["null"], + "type": "fill" + } + ], + "legendFormat": "__auto", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "refId": "A" + } + ], + "title": "REGISTRY-PROXY", + "type": "row" + } + ], + "refresh": "5s", + "schemaVersion": 37, + "style": "dark", + "tags": ["drycc"], + "templating": { + "list": [] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "browser", + "title": "Drycc Component Health", + "uid": "2ySBrTTMz", + "version": 1, + "weekStart": "" +} diff --git a/rootfs/usr/share/grafana/entrypoint.sh b/rootfs/usr/share/grafana/entrypoint.sh deleted file mode 100644 index 5fc4448..0000000 --- a/rootfs/usr/share/grafana/entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -exec "$@" diff --git a/rootfs/usr/share/grafana/grafana.ini.tpl b/rootfs/usr/share/grafana/grafana.ini.tpl deleted file mode 100644 index 31a258a..0000000 --- a/rootfs/usr/share/grafana/grafana.ini.tpl +++ /dev/null @@ -1,261 +0,0 @@ -#################### Grafana Configuration Example ##################### -# -# Everything has defaults so you only need to uncomment things you want to -# change - -# possible values : production, development -app_mode = {{ default "development" .APP_MODE }} - -#################################### Paths #################################### -[paths] -# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) -# -data = {{ default "/var/lib/grafana" .DATA_PATH }} - -# -# Directory where grafana can store logs -# -logs = {{ default "/var/log" .LOG_PATH }} - -# -# Directory where grafana will automatically scan and look for plugins -# -plugins = {{ default "/var/lib/grafana/plugins" .PLUGINS_PATH }} - -#################################### Server #################################### -[server] -# Protocol (http or https) -protocol = {{ default "http" .SERVER_PROTOCOL }} - -# The ip address to bind to, empty will bind to all interfaces -http_addr = {{ default "" .BIND_ADDRESS}} -# The http port to use -http_port = {{ default "3000" .BIND_PORT }} - -# The public facing domain name used to access grafana from a browser -domain = {{ default "localhost" .DOMAIN }} - -# Redirect to correct domain if host header does not match domain -# Prevents DNS rebinding attacks -enforce_domain = {{ default "false" .ENFORCE_DOMAIN }} - -# The full public facing url -root_url = {{ default "%(protocol)s://%(domain)s:%(http_port)s/" .ROOT_URL }} - -# Log web requests -router_logging = {{ default "false" .ROUTER_LOGGING }} - -# the path relative working path -static_root_path = {{ default "public" .STATIC_ROOT_PATH }} - -# enable gzip -enable_gzip = {{ default "false" .ENABLE_GZIP }} - -# https certs & key file -{{ if .CERT_FILE }} -cert_file = {{ .CERT_FILE }} -{{ end }} -{{ if .CERT_KEY }} -cert_key = {{ .CERT_KEY }} -{{ end }} - -#################################### Database #################################### -[database] -# Either "mysql", "postgres" or "sqlite3", it's your choice -{{ if .GF_DATABASE_TYPE }} -type = {{ .GF_DATABASE_TYPE }} -{{ end }} -{{ if .GF_DATABASE_URL }} -url = {{ .GF_DATABASE_URL }} -{{ end }} - -#################################### Analytics #################################### -[analytics] -# Server reporting, sends usage counters to stats.grafana.org every 24 hours. -# No ip addresses are being tracked, only simple counters to track -# running instances, dashboard and error counts. It is very helpful to us. -# Change this option to false to disable reporting. -reporting_enabled = {{ default "false" .GRAFANA_REPORTING }} - -# Google Analytics universal tracking code, only enabled if you specify an id here -{{ if .GOOGLE_ANALYTICS_UA_ID }} -google_analytics_ua_id = {{ .GOOGLE_ANALYTICS_UA_ID }} -{{ end }} - -#################################### Security #################################### -[security] -# default admin user, created on startup -admin_user = {{ default "admin" .DEFAULT_USER }} - -# default admin password, can be changed before first start of grafana, or in profile settings -admin_password = {{ default "admin" .DEFAULT_USER_PASSWORD }} - -# used for signing -secret_key = {{ default "SW2YcwTIb9zpOOhoPsMm" .SECRET_KEY }} - -# Auto-login remember days -login_remember_days = {{ default "7" .LOGIN_REMEMBER_DAYS }} -cookie_username = {{ default "grafana_user" .COOKIE_USERNAME }} -cookie_remember_name = {{ default "grafana_remember" .COOKIE_REMEMBER_NAME }} - -# disable gravatar profile images -disable_gravatar = {{ default "false" .DISABLE_GRAVATAR }} - -# data source proxy whitelist (ip_or_domain:port seperated by spaces) -{{ if .DATASOURCE_PROXY_WHITELIST }} -data_source_proxy_whitelist = {{ .DATA_SOURCE_PROXY_WHITELIST }} -{{ end }} - -#################################### Users #################################### -[users] -# disable user signup / registration -allow_sign_up = {{ default "true" .ALLOW_SIGN_UP }} - -# Allow non admin users to create organizations -allow_org_create = {{ default "true" .ALLOW_ORG_CREATE }} - -# Set to true to automatically assign new users to the default organization (id 1) -auto_assign_org = {{ default "true" .AUTO_ASSIGN_ORG }} - -# Default role new users will be automatically assigned (if disabled above is set to true) -auto_assign_org_role = {{ default "Viewer" .AUTO_ASSIGN_ORG_ROLE }} - -# Background text for the user field on the login page -login_hint = {{ default "email or username" .LOGIN_HINT }} - -#################################### Anonymous Auth ########################## -[auth.anonymous] -# enable anonymous access -enabled = {{ default "false" .ANON_ACCESS }} - -# specify organization name that should be used for unauthenticated users -org_name = {{ default "Main Org." .ORG_NAME }} - -# specify role for unauthenticated users -org_role = {{ default "Viewer" .UNAUTHED_USER_ORG_ROLE }} - -#################################### Github Auth ########################## -[auth.github] -{{ if .GITHUB_AUTH }} -enabled = {{ .GITHUB_AUTH }} -allow_sign_up = {{ default "false" .GITHUB_AUTH_ALLOW_SIGN_UP }} -client_id = {{ .GITHUB_AUTH_CLIENT_ID }} -client_secret = {{ .GITHUB_AUTH_CLIENT_SECRET }} -scopes = {{ default "user:email,read:org" .GITHUB_AUTH_SCOPES }} -auth_url = {{ default "https://github.com/login/oauth/authorize" .GITHUB_AUTH_URL }} -token_url = {{ default "https://github.com/login/oauth/access_token" .GITHUB_AUTH_TOKEN_URL }} -api_url = {{ default "https://api.github.com/user" .GITHUB_AUTH_API_URL }} -team_ids = {{ default "" .GITHUB_AUTH_TEAM_IDS }} -allowed_organizations = {{ default "" .GITHUB_AUTH_ALLOWED_ORGS}} -{{ end }} - -#################################### Google Auth ########################## -[auth.google] -{{ if .GOOGLE_AUTH }} -enabled = {{ .GOOGLE_AUTH }} -allow_sign_up = {{ default "false" .GOOGLE_AUTH_ALLOW_SIGN_UP }} -client_id = {{ .GOOGLE_AUTH_CLIENT_ID }} -client_secret = {{ .GOOGLE_AUTH_CLIENT_SECRET }} -scopes = {{ default "https://www.googleapis.com/auth/userinfo.profile ttps://www.googleapis.com/auth/userinfo.email" .GOOGLE_AUTH_SCOPES }} -auth_url = {{ default "https://accounts.google.com/o/oauth2/auth" .GOOGLE_AUTH_URL }} -token_url = {{ default "https://accounts.google.com/o/oauth2/token" .GOOGLE_AUTH_TOKEN_URL }} -api_url = {{ default "https://www.googleapis.com/oauth2/v1/userinfo" .GOOGLE_AUTH_API_URL }} -allowed_domains = {{ default "" .GOOGLE_AUTH_ALLOWED_DOMAINS }} -{{ end }} - -#################################### Auth Proxy ########################## -[auth.proxy] -{{ if .AUTH_PROXY }} -enabled = {{ .AUTH_PROXY }} -header_name = {{ default "X-WEBAUTH-USER" .AUTH_HEADER_NAME }} -header_property = {{ default "username" .AUTH_HEADER_PROPERTY }} -auto_sign_up = {{ default "true" .AUTH_AUTO_SIGN_UP }} -{{ end }} - -#################################### Basic Auth ########################## -[auth.basic] -enabled = {{ default "true" .BASIC_AUTH }} - -#################################### Auth LDAP ########################## -[auth.ldap] -{{ if .LDAP_ENDPOINT }} -enabled = true -config_file = "/usr/share/grafana/ldap.toml" -{{ end }} - -#################################### Generic Oauth ########################## -[auth.generic_oauth] -{{ if .DRYCC_PASSPORT_URL }} -name = OAuth -enabled = true -auto_login = true -client_id = {{ .DRYCC_PASSPORT_KEY }} -client_secret = {{ .DRYCC_PASSPORT_SECRET }} -scopes = profile,openid -auth_url = {{ .DRYCC_PASSPORT_URL }}/oauth/authorize/ -token_url = {{ .DRYCC_PASSPORT_URL }}/oauth/token/ -api_url = {{ .DRYCC_PASSPORT_URL }}/oauth/userinfo/ -allow_sign_up = true -tls_skip_verify_insecure = true -role_attribute_path = (is_superuser && 'Admin') || (is_staff && 'Viewer') || '' -role_attribute_strict = true - -[server] -root_url = {{ .DRYCC_GRAFANA_URL }} - -{{ end }} - -#################################### SMTP / Emailing ########################## -[smtp] -{{ if .SMTP }} -enabled = {{ .SMTP }} -host = {{ .SMTP_HOST }} -user = {{ .SMTP_USER }} -password = {{ .SMTP_PASSWORD }} -cert_file = {{ .SMTP_CERT_FILE }} -key_file = {{ .SMTP_KEY_FILE }} -skip_verify = {{ default "false" .SMTP_SKIP_VERIFY }} -from_address = {{ default "admin@grafana.localhost" .SMTP_FROM_ADDRESS }} -{{ end }} - -[emails] -welcome_email_on_sign_up = {{ default "false" .WELCOME_EMAIL }} - -#################################### Logging ########################## -[log] -# Either "console", "file", default is "console" -# Use comma to separate multiple modes, e.g. "console, file" -mode = {{ default "console" .LOG_MODE }} - -# Buffer length of channel, keep it as it is if you don't know what it is. -buffer_len = {{ default "10000" .BUFFER_LENGTH }} - -# Either "Trace", "Debug", "Info", "Warn", "Error", "Critical", default is "Trace" -level = {{ default "Info" .LOG_LEVEL }} - -# For "console" mode only -[log.console] -level = {{ default "Info" .LOG_LEVEL_CONSOLE }} - -# For "file" mode only -[log.file] -level = {{ default "Info" .LOG_LEVEL_FILE }} -# This enables automated log rotate(switch of following options), default is true -log_rotate = {{ default "true" .LOG_ROTATE }} - -# Max line number of single file, default is 1000000 -max_lines = {{ default "1000000" .LOG_MAX_LINES }} - -# Max size shift of single file, default is 28 means 1 << 28, 256MB -max_lines_shift = {{ default "28" .LOG_MAX_LINES_SHIFT }} - -# Segment log daily, default is true -daily_rotate = {{ default "true" .LOG_DAILY_ROTATE }} - -# Expired days of log file(delete after max days), default is 7 -max_days = {{ default "7" .LOG_MAX_DAYS }} - -;#################################### Dashboard JSON files ########################## -[dashboards.json] -enabled = {{ default "true" .DASHBOARD_JSON }} -path = {{ default "/usr/share/grafana/dashboards" .DASHBOARD_JSON_PATH }} diff --git a/rootfs/usr/share/grafana/ldap.toml.tpl b/rootfs/usr/share/grafana/ldap.toml.tpl deleted file mode 100644 index 1c8b617..0000000 --- a/rootfs/usr/share/grafana/ldap.toml.tpl +++ /dev/null @@ -1,45 +0,0 @@ -[[servers]] -# Ldap server host (specify multiple hosts space separated) -host = "{{ .LDAP_HOST }}" -# Default port is 389 or 636 if use_ssl = true -port = {{ .LDAP_PORT }} -# Set to true if LDAP server supports TLS -use_ssl = {{ .LDAP_USE_SSL }} -# Set to true if connect LDAP server with STARTTLS pattern (create connection in insecure, then upgrade to secure connection with TLS) -start_tls = true -# set to true if you want to skip SSL cert validation -ssl_skip_verify = true - -# Search user bind dn -bind_dn = "{{ .LDAP_BIND_DN }}" -# Search user bind password -# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" -bind_password = "{{ .LDAP_BIND_PASSWORD }}" - -# User search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)" -# Allow login from email or username, example "(|(sAMAccountName=%s)(userPrincipalName=%s))" -search_filter = "{{ .LDAP_USER_FILTER }}" - -# An array of base dns to search through -search_base_dns = ["{{ .LDAP_USER_BASEDN }}"] - -# Specify names of the LDAP attributes your LDAP uses -[servers.attributes] -name = "givenName" -surname = "sn" -username = "cn" -member_of = "memberOf" -email = "mail" - -[[servers.group_mappings]] -group_dn = "{{ .LDAP_ADMIN_GROUP }}" -org_role = "Admin" -grafana_admin = true # Available in Grafana v5.3 and above - -[[servers.group_mappings]] -group_dn = "{{ .LDAP_EDITOR_GROUP }}" -org_role = "Editor" - -[[servers.group_mappings]] -group_dn = "{{ .LDAP_VIEWER_GROUP }}" -org_role = "Viewer" diff --git a/rootfs/usr/share/grafana/start-grafana b/rootfs/usr/share/grafana/start-grafana deleted file mode 100755 index f9b42b7..0000000 --- a/rootfs/usr/share/grafana/start-grafana +++ /dev/null @@ -1,140 +0,0 @@ -#!/bin/bash - -# Some of the contents of this file came from here - https://github.com/kubernetes/heapster/blob/ed5baadf04ea9f8e48fc7d44dad63a63af34ff9b/grafana/run.sh -HEADER_CONTENT_TYPE="Content-Type: application/json" -HEADER_ACCEPT="Accept: application/json" - -NAMESPACE=${NAMESPACE:-drycc} -GRAFANA_USER=${DEFAULT_USER:-admin} -GRAFANA_PASSWD=${DEFAULT_USER_PASSWORD:-admin} -GRAFANA_PORT=${BIND_PORT:-3000} - -DASHBOARD_LOCATION=${DASHBOARD_JSON_PATH:-"/usr/share/grafana/api/dashboards"} -# Create the dashboards directory -mkdir /usr/share/grafana/dashboards - -echo "Building grafana.ini!" -envtpl > grafana.ini < grafana.ini.tpl -echo "Finished building grafana config..." -echo "###########################################" -echo "###########################################" -cat /usr/share/grafana/grafana.ini -echo "###########################################" -echo "###########################################" - -if [ -n "${LDAP_ENDPOINT}" ]; then - echo "Building ldap.toml!" - OLD_IFS="$IFS" - IFS=':/' read -ra ldap_url_array <<< "$LDAP_ENDPOINT" - IFS="$OLD_IFS" - ldap_schema=${ldap_url_array[0]} - if [ "${ldap_schema}" == "ldaps" ]; then - export LDAP_USE_SSL=true - else - export LDAP_USE_SSL=false - fi - export LDAP_HOST=${ldap_url_array[3]//\/\//} - export LDAP_PORT=${ldap_url_array[4]} - envtpl > ldap.toml < ldap.toml.tpl - echo "Finished building ldap config..." - echo "###########################################" - echo "###########################################" - cat /usr/share/grafana/ldap.toml - echo "###########################################" - echo "###########################################" -fi - -GF_PATHS_PLUGINS=${PLUGINS_PATH:-"/var/lib/grafana/plugins"} -if [ -n "${GF_INSTALL_PLUGINS}" ]; then - echo "Installing Grafana plugins..." - OLDIFS=$IFS - IFS=',' - for plugin in ${GF_INSTALL_PLUGINS}; do - echo "Installing ${plugin} ..." - grafana-cli --pluginsDir "${GF_PATHS_PLUGINS}" plugins install "${plugin}" - echo "" - echo "Done installing ${plugin}" - done - IFS=$OLDIFS -fi - -set -m -echo "Starting Grafana in the background" -exec grafana server -config /usr/share/grafana/grafana.ini -homepath /opt/drycc/grafana & -echo "Waiting for Grafana to come up..." -until curl -q --fail --output /dev/null --silent "http://${GRAFANA_USER}:${GRAFANA_PASSWD}@localhost:${GRAFANA_PORT}/api/org"; do - printf "." - sleep 2 -done -echo "Grafana is up and running." - -# Allow access to dashboards without having to log in -export GF_AUTH_ANONYMOUS_ENABLED=true -export GF_SERVER_HTTP_PORT=${GRAFANA_PORT} - -BACKEND_ACCESS_MODE=${BACKEND_ACCESS_MODE:-proxy} -PROMETHEUS_URL=${PROMETHEUS_URL:-"http://${DRYCC_PROMETHEUS_SERVICE_HOST}:${DRYCC_PROMETHEUS_SERVICE_PORT}"} - -echo "Using the following URL for prometheus: ${PROMETHEUS_URL}" -echo "Using the following backend access mode for prometheus: ${BACKEND_ACCESS_MODE}" - -echo "Creating default prometheus datasource..." -curl -i -XPOST -H "${HEADER_ACCEPT}" -H "${HEADER_CONTENT_TYPE}" "http://${GRAFANA_USER}:${GRAFANA_PASSWD}@localhost:${GRAFANA_PORT}/api/datasources" -d ' -{ - "name": "prometheus-datasource", - "type": "prometheus", - "access": "'"${BACKEND_ACCESS_MODE}"'", - "isDefault": true, - "url": "'"${PROMETHEUS_URL}"'" -}' - -echo "" -echo "Importing default dashboards..." -# replace namespace -# shellcheck disable=SC1003 -# shellcheck disable=SC2046 -sed -i 's@r\[\\"namespace\\\"\] == \\"drycc\\"@r\[\\"namespace\\"\] == \\"\'"${NAMESPACE}"'\\"@g' $(grep -F 'r[\"namespace\"] == \"drycc\"' -rl --include="*.json" "${DASHBOARD_LOCATION}") -# drycc component dashboard -DCD="${DASHBOARD_LOCATION}"/drycc_component_health.json - -# remove off-cluster component panel -if [ "on-cluster" != "${VALKEY_LOCATION}" ]; then - rm -rf "${DASHBOARD_LOCATION}"/drycc_valkey.json - # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "VALKEY"))')" > "${DCD}" -fi -if [ "on-cluster" != "${DATABASE_LOCATION}" ]; then - rm -rf "${DASHBOARD_LOCATION}"/drycc_database.json - # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "DATABASE"))')" > "${DCD}" -fi -if [ "on-cluster" != "${PASSPORT_LOCATION}" ]; then - # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "PASSPORT"))')" > "${DCD}" -fi -if [ "on-cluster" != "${REGISTRY_LOCATION}" ]; then - rm -rf "${DASHBOARD_LOCATION}"/drycc_registry.json - # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "REGISTRY"))')" > "${DCD}" - # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "REGISTRY-PROXY"))')" > "${DCD}" -fi -if [ "on-cluster" != "${STORAGE_LOCATION}" ]; then - rm -rf "${DASHBOARD_LOCATION}"/drycc_storage.json - # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "STORAGE"))')" > "${DCD}" -fi - -if [ "on-cluster" != "${PROMETHEUS_LOCATION}" ]; then - rm -rf "${DASHBOARD_LOCATION}"/drycc_victoriametrics_*.json -fi - -for filename in "${DASHBOARD_LOCATION}"/*.json; do - echo "Importing ${filename} ..." - curl -i -XPOST --data "@${filename}" -H "${HEADER_ACCEPT}" -H "${HEADER_CONTENT_TYPE}" "http://${GRAFANA_USER}:${GRAFANA_PASSWD}@localhost:${GRAFANA_PORT}/api/dashboards/db" - echo "" - echo "Done importing ${filename}" -done - -echo "Bringing Grafana back to the foreground" -fg From 022786df89c682ad7424f712edefbc36c26905d3 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 10 Jun 2025 22:54:29 +0800 Subject: [PATCH 24/50] feat(grafana): add oauth2 proxy --- Makefile | 6 +- charts/grafana/templates/_helpers.tmpl | 120 +- .../grafana/templates/grafana-configmap.yaml | 86 +- .../grafana/templates/grafana-deployment.yaml | 141 +- .../templates/grafana-secret-creds.yaml | 10 + charts/grafana/templates/grafana-svc.yaml | 2 +- charts/grafana/values.yaml | 14 +- rootfs/Dockerfile | 13 +- rootfs/usr/bin/start-grafana | 108 - .../api/dashboards/user/user_container.json | 2770 ------------- .../drycc_component_health.json | 50 +- .../main => dashboards}/drycc_database.json | 14 +- .../main => dashboards}/drycc_registry.json | 2 +- .../main => dashboards}/drycc_storage.json | 2 +- .../main => dashboards}/drycc_valkey.json | 18 +- .../drycc_victoriametrics_cluster.json | 2 +- .../drycc_victoriametrics_vmagent.json | 2 +- .../drycc_victoriametrics_vmauth.json | 2576 ++++++++++++ .../main => dashboards}/kubernetes.json | 2 +- .../kubernetes_apiserver.json | 2 +- .../kubernetes_cluster.json | 70 +- .../kubernetes_deployment_metrics.json | 12 +- .../kubernetes_metrics.json | 2 +- .../dashboards/main => dashboards}/node.json | 4 +- rootfs/usr/share/grafana/oauth2/__init__.py | 0 .../share/grafana/oauth2/alerting/test.json | 74 + .../share/grafana/oauth2/dashboard/node.json | 3613 +++++++++++++++++ .../usr/share/grafana/oauth2/hook/__init__.py | 7 + .../hook/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 327 bytes .../hook/__pycache__/grafana.cpython-312.pyc | Bin 0 -> 10656 bytes .../usr/share/grafana/oauth2/hook/grafana.py | 178 + rootfs/usr/share/grafana/oauth2/main.py | 95 + .../usr/share/grafana/oauth2/requirements.txt | 8 + .../grafana/provisioning/alerting/drycc.yaml | 1 + .../provisioning/dashboards/drycc.yaml | 7 + .../provisioning/datasources/drycc.yaml | 14 + .../grafana/provisioning/plugins/drycc.yaml | 1 + 37 files changed, 6959 insertions(+), 3067 deletions(-) create mode 100644 charts/grafana/templates/grafana-secret-creds.yaml delete mode 100755 rootfs/usr/bin/start-grafana delete mode 100644 rootfs/usr/share/grafana/api/dashboards/user/user_container.json rename rootfs/usr/share/grafana/{api/dashboards/main => dashboards}/drycc_component_health.json (98%) rename rootfs/usr/share/grafana/{api/dashboards/main => dashboards}/drycc_database.json (99%) rename rootfs/usr/share/grafana/{api/dashboards/main => dashboards}/drycc_registry.json (99%) rename rootfs/usr/share/grafana/{api/dashboards/main => dashboards}/drycc_storage.json (99%) rename rootfs/usr/share/grafana/{api/dashboards/main => dashboards}/drycc_valkey.json (99%) rename rootfs/usr/share/grafana/{api/dashboards/main => dashboards}/drycc_victoriametrics_cluster.json (99%) rename rootfs/usr/share/grafana/{api/dashboards/main => dashboards}/drycc_victoriametrics_vmagent.json (99%) create mode 100644 rootfs/usr/share/grafana/dashboards/drycc_victoriametrics_vmauth.json rename rootfs/usr/share/grafana/{api/dashboards/main => dashboards}/kubernetes.json (99%) rename rootfs/usr/share/grafana/{api/dashboards/main => dashboards}/kubernetes_apiserver.json (99%) rename rootfs/usr/share/grafana/{api/dashboards/main => dashboards}/kubernetes_cluster.json (98%) rename rootfs/usr/share/grafana/{api/dashboards/main => dashboards}/kubernetes_deployment_metrics.json (99%) rename rootfs/usr/share/grafana/{api/dashboards/main => dashboards}/kubernetes_metrics.json (99%) rename rootfs/usr/share/grafana/{api/dashboards/main => dashboards}/node.json (99%) create mode 100644 rootfs/usr/share/grafana/oauth2/__init__.py create mode 100644 rootfs/usr/share/grafana/oauth2/alerting/test.json create mode 100644 rootfs/usr/share/grafana/oauth2/dashboard/node.json create mode 100644 rootfs/usr/share/grafana/oauth2/hook/__init__.py create mode 100644 rootfs/usr/share/grafana/oauth2/hook/__pycache__/__init__.cpython-312.pyc create mode 100644 rootfs/usr/share/grafana/oauth2/hook/__pycache__/grafana.cpython-312.pyc create mode 100644 rootfs/usr/share/grafana/oauth2/hook/grafana.py create mode 100644 rootfs/usr/share/grafana/oauth2/main.py create mode 100644 rootfs/usr/share/grafana/oauth2/requirements.txt create mode 100644 rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml create mode 100644 rootfs/usr/share/grafana/provisioning/dashboards/drycc.yaml create mode 100644 rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml create mode 100644 rootfs/usr/share/grafana/provisioning/plugins/drycc.yaml diff --git a/Makefile b/Makefile index c10cfc8..b964ce2 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ SHELL := /bin/bash # grafana share/use the following targets/exports SHORT_NAME ?= grafana -SHELL_SCRIPTS = rootfs/usr/bin/start-grafana +SHELL_SCRIPTS = rootfs/usr/share/grafana/oauth2 BUILD_TAG ?= git-$(shell git rev-parse --short HEAD) DRYCC_REGISTRY ?= ${DEV_REGISTRY} @@ -12,7 +12,7 @@ PLATFORM ?= linux/amd64,linux/arm64 include includes.mk include versioning.mk -TEST_ENV_PREFIX := podman run --rm -v ${CURDIR}:/bash -w /bash ${DEV_REGISTRY}/drycc/go-dev +TEST_ENV_PREFIX := podman run --rm -v ${CURDIR}:/bash -w /bash ${DEV_REGISTRY}/drycc/python-dev build: podman-build push: podman-push @@ -31,7 +31,7 @@ clean: check-podman test: test-style test-style: - ${TEST_ENV_PREFIX} shellcheck $(SHELL_SCRIPTS) + ${TEST_ENV_PREFIX} flake8 --show-source --max-line-length=100 $(SHELL_SCRIPTS) .PHONY: build push podman-build clean upgrade deploy test test-style diff --git a/charts/grafana/templates/_helpers.tmpl b/charts/grafana/templates/_helpers.tmpl index 9869f8d..96c5370 100644 --- a/charts/grafana/templates/_helpers.tmpl +++ b/charts/grafana/templates/_helpers.tmpl @@ -9,31 +9,60 @@ env: valueFrom: fieldRef: fieldPath: metadata.namespace -{{- if .Values.prometheusUrl }} -- name: "PROMETHEUS_URL" - value: "{{ .Values.prometheusUrl }}" -{{- else if .Values.prometheus.enabled }} -- name: "PROMETHEUS_URL" - value: "http://$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)/v2/prometheus" +{{- if (.Values.valkeyUrl) }} +- name: DRYCC_VALKEY_URL + value: "{{ .Values.valkeyUrl }}" +{{- else if .Values.valkey.enabled }} +- name: DRYCC_VALKEY_PASSWORD + valueFrom: + secretKeyRef: + name: valkey-creds + key: password +- name: DRYCC_VALKEY_URL + value: "redis://:$(DRYCC_VALKEY_PASSWORD)@drycc-valkey.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:16379/0" {{- end }} -- name: "VALKEY_LOCATION" - value: {{ ternary "on-cluster" "off-cluster" .Values.valkey.enabled }} -- name: "STORAGE_LOCATION" - value: {{ ternary "on-cluster" "off-cluster" .Values.storage.enabled }} -- name: "DATABASE_LOCATION" - value: {{ ternary "on-cluster" "off-cluster" .Values.database.enabled }} -- name: "PASSPORT_LOCATION" - value: {{ ternary "on-cluster" "off-cluster" .Values.passport.enabled }} -- name: "REGISTRY_LOCATION" - value: {{ ternary "on-cluster" "off-cluster" .Values.registry.enabled }} -- name: "PROMETHEUS_LOCATION" - value: {{ ternary "on-cluster" "off-cluster" .Values.prometheus.enabled }} -- name: "DRYCC_GATEWAY_SCHEME" +{{- if .Values.victoriametricsUrl }} +- name: "DRYCC_VICTORIAMETRICS_URL" + value: "{{ .Values.victoriametricsUrl }}" +{{- else if .Values.victoriametrics.enabled }} +- name: "DRYCC_VICTORIAMETRICS_USERNAME" + valueFrom: + secretKeyRef: + name: victoriametrics-vmauth-creds + key: username +- name: "DRYCC_VICTORIAMETRICS_PASSWORD" + valueFrom: + secretKeyRef: + name: victoriametrics-vmauth-creds + key: password +- name: "DRYCC_VICTORIAMETRICS_URL" + value: "http://$(DRYCC_VICTORIAMETRICS_USERNAME):$(DRYCC_VICTORIAMETRICS_PASSWORD)@drycc-victoriametrics-vmauth.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:8427/select/0/prometheus" +{{- end }} +{{- if .Values.passport.enabled}} +- name: "DRYCC_PASSPORT_URL" {{- if .Values.global.certManagerEnabled }} - value: https + value: https://drycc-passport.{{ .Values.global.platformDomain }} {{- else }} - value: http -{{- end}} + value: http://drycc-passport.{{ .Values.global.platformDomain }} +{{- end }} +- name: DRYCC_PASSPORT_KEY + valueFrom: + secretKeyRef: + name: passport-creds + key: drycc-passport-grafana-key +- name: DRYCC_PASSPORT_SECRET + valueFrom: + secretKeyRef: + name: passport-creds + key: drycc-passport-grafana-secret +{{- else }} +- name: DRYCC_PASSPORT_URL + value: "{{ .Values.passportUrl }}" +- name: DRYCC_PASSPORT_KEY + value: "{{ .Values.passportKey }}" +- name: DRYCC_PASSPORT_SECRET + value: "{{ .Values.passportSecret }}" +{{- end }} - name: GF_DATABASE_TYPE value: postgres {{- if (.Values.databaseUrl) }} @@ -53,49 +82,22 @@ env: - name: GF_DATABASE_URL value: "postgres://$(GF_DATABASE_USER):$(GF_DATABASE_PASSWORD)@drycc-database.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:5432/grafana" {{- end }} -{{- if not (.Values.environment.GF_SECURITY_ADMIN_USER) }} - name: "GF_SECURITY_ADMIN_USER" - value: {{ randAlphaNum 32 }} -{{- end}} -{{- if not (.Values.environment.GF_SECURITY_ADMIN_PASSWORD) }} -- name: "GF_SECURITY_ADMIN_PASSWORD" - value: {{ randAlphaNum 32 }} -{{- end}} -{{- if not (.Values.environment.GF_SECURITY_SECRET_KEY) }} -- name: "GF_SECURITY_SECRET_KEY" - value: {{ randAlphaNum 32 }} -{{- end}} -- name: "GF_SERVER_ROOT_URL" - value: $(DRYCC_GATEWAY_SCHEME)://drycc-grafana.{{ .Values.global.platformDomain }} -{{- if .Values.passport.enabled}} -- name: GF_AUTH_GENERIC_OAUTH_AUTH_URL - value: $(DRYCC_GATEWAY_SCHEME)://drycc-passport.{{ .Values.global.platformDomain }}/oauth/authorize/ -- name: GF_AUTH_GENERIC_OAUTH_TOKEN_URL - value: $(DRYCC_GATEWAY_SCHEME)://drycc-passport.{{ .Values.global.platformDomain }}/oauth/token/ -- name: GF_AUTH_GENERIC_OAUTH_API_URL - value: $(DRYCC_GATEWAY_SCHEME)://drycc-passport.{{ .Values.global.platformDomain }}/oauth/userinfo/ -- name: GF_AUTH_GENERIC_OAUTH_CLIENT_ID valueFrom: secretKeyRef: - name: passport-creds - key: drycc-passport-grafana-key -- name: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET + name: grafana-creds + key: admin-username +- name: "GF_SECURITY_ADMIN_PASSWORD" valueFrom: secretKeyRef: - name: passport-creds - key: drycc-passport-grafana-secret + name: grafana-creds + key: admin-password +- name: "GF_SERVER_ROOT_URL" +{{- if .Values.global.certManagerEnabled }} + value: https://drycc-grafana.{{ .Values.global.platformDomain }} {{- else }} -- name: GF_AUTH_GENERIC_OAUTH_AUTH_URL - value: {{ .Values.passportUrl }}/oauth/authorize/ -- name: GF_AUTH_GENERIC_OAUTH_TOKEN_URL - value: {{ .Values.passportUrl }}/oauth/token/ -- name: GF_AUTH_GENERIC_OAUTH_API_URL - value: {{ .Values.passportUrl }}/oauth/userinfo/ -- name: GF_AUTH_GENERIC_OAUTH_CLIENT_ID - value: "{{ .Values.passportKey }}" -- name: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET - value: "{{ .Values.passportSecret }}" -{{- end }} + value: http://drycc-grafana.{{ .Values.global.platformDomain }} +{{- end}} - name: GF_UNIFIED_ALERTING_HA_PEERS value: "drycc-grafana.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:9094" - name: GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS diff --git a/charts/grafana/templates/grafana-configmap.yaml b/charts/grafana/templates/grafana-configmap.yaml index 7b761a7..f741ae9 100644 --- a/charts/grafana/templates/grafana-configmap.yaml +++ b/charts/grafana/templates/grafana-configmap.yaml @@ -6,6 +6,33 @@ metadata: labels: heritage: drycc data: + Caddyfile: | + { + order reverse_proxy before encode + auto_https off + log { + output stdout + } + } + :5000 { + handle /oauth2/* { + reverse_proxy 127.0.0.1:4000 + } + + handle { + forward_auth 127.0.0.1:4000 { + uri /oauth2/userinfo + copy_headers Remote-User Remote-Name Remote-Email + header_up -Connection + header_up X-Real-IP {remote_host} + @error status 401 + handle_response @error { + redir * /oauth2/sign_in?redirect={scheme}://{host}{uri} + } + } + reverse_proxy 127.0.0.1:3000 + } + } grafana.ini: | app_mode = production @@ -13,8 +40,8 @@ data: data = /var/lib/grafana home = /usr/share/grafana logs = /var/log/grafana - plugins = /var/lib/grafana/plugins - provisioning = /opt/drycc/grafana/conf/provisioning + plugins = /usr/share/grafana/plugins + provisioning = /usr/share/grafana/provisioning [server] http_port = 3000 @@ -26,9 +53,8 @@ data: reporting_enabled = false [security] - admin_user = drycc - admin_password = drycc admin_email = admin@drycc.cc + secret_key = KSJRZP0RIhO7P14e1TykTEzWuJhBobAj login_remember_days = 7 cookie_username = grafana_user cookie_remember_name = grafana_remember @@ -37,31 +63,31 @@ data: [users] allow_sign_up = false allow_org_create = false - auto_assign_org = true - auto_assign_org_id = 1 + auto_assign_org = false login_hint = email or username [auth] disable_login_form = true - - [auth.anonymous] - enabled = false - org_name = Main Org. + disable_signout_menu = false [auth.basic] enabled = true - [auth.generic_oauth] + [auth.proxy] enabled = true - name = Drycc OAuth - auto_login = true - scopes = email,profile,openid - allow_sign_up = true - tls_skip_verify_insecure = true - org_attribute_path = roles - org_mapping = admin:*:Admin staff:*:Editor users:2:Viewer - role_attribute_path = (is_active && ((is_superuser && 'GrafanaAdmin') || (is_staff && 'Editor'))) || 'None' - allow_assign_grafana_admin = true + header_name = Remote-User + header_property = username + auto_sign_up = true + sync_ttl = 15 + headers = Name:Remote-Name Email:Remote-Email + enable_login_token = false + + [live] + max_connections = 100 + message_size_limit = 65536 + allowed_origins = * + ha_engine = redis + ha_prefix = "grafana:live" [emails] welcome_email_on_sign_up = false @@ -74,13 +100,25 @@ data: [log.console] level = info + [quota] + enabled = true + org_user = 2 + org_dashboard = 20 + org_data_source = 5 + org_api_key = 2 + org_alert_rule = 20 + user_org = 2 + alerting_rule_group_rules = 20 + alerting_rule_evaluation_results = 20 + [unified_alerting] enabled = true - disabled_orgs = 2 ha_peer_timeout = 15s ha_reconnect_timeout = 2m ha_listen_address = 0.0.0.0:9094 + evaluation_timeout = 3s + max_attempts = 3 + min_interval = 1m - [dashboards.json] - enabled = true - path = /usr/share/grafana/dashboards + [plugins] + plugin_admin_enabled = false diff --git a/charts/grafana/templates/grafana-deployment.yaml b/charts/grafana/templates/grafana-deployment.yaml index b3fc564..c71566c 100644 --- a/charts/grafana/templates/grafana-deployment.yaml +++ b/charts/grafana/templates/grafana-deployment.yaml @@ -46,6 +46,23 @@ spec: {{- if .Values.diagnosticMode.enabled }} command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }} args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }} + {{- else }} + args: + - /usr/bin/env + - bash + - -ec + - | + set -m + echo "Configure Grafana live HA engine" + DRYCC_VALKEY_JSON=$(echo "$DRYCC_VALKEY_URL" | jq -R 'capture("^(?redis[s]?)://(?[^:]+)?(?::(?[^@]+))?@(?
[^/]+)/?(?[0-9]*)$")') + GF_LIVE_HA_ENGINE_ADDRESS=$(echo "${DRYCC_VALKEY_JSON}" |jq -r '.address') + GF_LIVE_HA_ENGINE_PASSWORD=$(echo "${DRYCC_VALKEY_JSON}" |jq -r '.password') + DRYCC_VICTORIAMETRICS_JSON=$(echo "$DRYCC_VICTORIAMETRICS_URL" | jq -R 'capture("^(?https?://)(?[^:@]+):(?[^@]+)@(?
.+)$") | {url: "\(.scheme)\(.address)", username: .username, password: .password}') + DRYCC_VICTORIAMETRICS_URL=$(echo "${DRYCC_VICTORIAMETRICS_JSON}" |jq -r '.url') + DRYCC_VICTORIAMETRICS_USERNAME=$(echo "${DRYCC_VICTORIAMETRICS_JSON}" |jq -r '.username') + DRYCC_VICTORIAMETRICS_PASSWORD=$(echo "${DRYCC_VICTORIAMETRICS_JSON}" |jq -r '.password') + export GF_LIVE_HA_ENGINE_ADDRESS GF_LIVE_HA_ENGINE_PASSWORD DRYCC_VICTORIAMETRICS_URL DRYCC_VICTORIAMETRICS_USERNAME DRYCC_VICTORIAMETRICS_PASSWORD + exec grafana server --config /usr/share/grafana/grafana.ini --homepath /opt/drycc/grafana {{- end }} {{- with index .Values "resources" }} resources: @@ -57,6 +74,7 @@ spec: - containerPort: 9094 name: alerting {{- include "grafana.envs" . | indent 8 }} + {{- if not .Values.diagnosticMode.enabled }} livenessProbe: tcpSocket: port: grafana @@ -78,25 +96,138 @@ spec: httpGet: path: /api/health port: grafana - initialDelaySeconds: 30 + initialDelaySeconds: 120 periodSeconds: 10 timeoutSeconds: 5 failureThreshold: 6 successThreshold: 1 - {{- if .Values.persistence.enabled }} + {{- end }} volumeMounts: + {{- if .Values.persistence.enabled }} - name: grafana-data mountPath: /var/lib/grafana + {{- end }} - name: grafana-config - readOnly: false subPath: grafana.ini readOnly: true - mountPath: /opt/drycc/grafana/conf/grafana.ini + mountPath: /usr/share/grafana/grafana.ini + - name: drycc-grafana-oauth2 + image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/grafana:{{.Values.imageTag}} + imagePullPolicy: {{.Values.imagePullPolicy}} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }} + {{- else }} + args: + - /usr/bin/env + - bash + - -ec + - | + set -m + python3 oauth2/main.py \ + --port 4000 \ + --client-id $(DRYCC_PASSPORT_KEY) \ + --client-secret $(DRYCC_PASSPORT_SECRET) \ + --oidc-issuer-url $(DRYCC_PASSPORT_URL)/oauth + {{- end }} + {{- with index .Values "resources" }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + ports: + - containerPort: 4000 + name: oauth2 + {{- include "grafana.envs" . | indent 8 }} + {{- if not .Values.diagnosticMode.enabled }} + livenessProbe: + tcpSocket: + port: oauth2 + initialDelaySeconds: 120 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + readinessProbe: + httpGet: + path: /oauth2/healthz + port: oauth2 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + startupProbe: + httpGet: + path: /oauth2/healthz + port: oauth2 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + {{- end }} + - name: drycc-grafana-proxy + image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/grafana:{{.Values.imageTag}} + imagePullPolicy: {{.Values.imagePullPolicy}} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }} + {{- else }} + args: + - /usr/bin/env + - bash + - -ec + - | + set -m + caddy run --config /usr/share/grafana/Caddyfile + {{- end }} + {{- with index .Values "resources" }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + ports: + - containerPort: 5000 + name: proxy + {{- include "grafana.envs" . | indent 8 }} + {{- if not .Values.diagnosticMode.enabled }} + livenessProbe: + tcpSocket: + port: proxy + initialDelaySeconds: 120 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + readinessProbe: + httpGet: + path: /oauth2/healthz + port: proxy + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + startupProbe: + httpGet: + path: /oauth2/healthz + port: proxy + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + {{- end }} + volumeMounts: + - name: grafana-config + subPath: Caddyfile + readOnly: true + mountPath: /usr/share/grafana/Caddyfile volumes: + {{- if .Values.persistence.enabled }} - name: grafana-data persistentVolumeClaim: claimName: drycc-grafana + {{- end }} - name: grafana-config configMap: name: grafana-config - {{- end }} diff --git a/charts/grafana/templates/grafana-secret-creds.yaml b/charts/grafana/templates/grafana-secret-creds.yaml new file mode 100644 index 0000000..1ea2b53 --- /dev/null +++ b/charts/grafana/templates/grafana-secret-creds.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: grafana-creds + labels: + heritage: drycc +type: Opaque +data: + admin-username: {{ include "common.secrets.lookup" (dict "secret" "grafana-creds" "key" "admin-username" "defaultValue" (default (randAlphaNum 16) .Values.adminUsername) "context" $) }} + admin-password: {{ include "common.secrets.lookup" (dict "secret" "grafana-creds" "key" "admin-password" "defaultValue" (default (randAlphaNum 32) .Values.adminPassword) "context" $) }} diff --git a/charts/grafana/templates/grafana-svc.yaml b/charts/grafana/templates/grafana-svc.yaml index 15dbdb1..01bb4a2 100644 --- a/charts/grafana/templates/grafana-svc.yaml +++ b/charts/grafana/templates/grafana-svc.yaml @@ -13,7 +13,7 @@ spec: ports: - port: 80 name: grafana - targetPort: grafana + targetPort: proxy - port: 9094 name: alerting targetPort: alerting diff --git a/charts/grafana/values.yaml b/charts/grafana/values.yaml index cec250b..ad89542 100644 --- a/charts/grafana/values.yaml +++ b/charts/grafana/values.yaml @@ -42,6 +42,10 @@ podAntiAffinityPreset: extraMatchLabels: app: "drycc-grafana" +# grafana security config +adminUsername: "" +adminPassword: "" + # Service service: # Provide any additional service annotations @@ -56,14 +60,16 @@ persistence: # this is usually a non required setting. environment: {} -## databaseUrl and databaseReplicaUrl are will no longer use the built-in database component +## valkeyUrl are will no longer use the built-in valkey component +valkeyUrl: "" +## databaseUrl are will no longer use the built-in database component databaseUrl: "" # The passportUrl, passportKey and passportSecret are will no longer use the built-in passport component passportUrl: "" passportKey: "" passportSecret: "" -# prometheusUrl is will no longer use the built-in prometheus component -prometheusUrl: "" +# victoriametricsUrl is will no longer use the built-in victoriametrics component +victoriametricsUrl: "" valkey: enabled: true @@ -85,7 +91,7 @@ registry: passport: enabled: true -prometheus: +victoriametrics: enabled: true global: diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 48fcbac..8be4e5d 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -5,7 +5,9 @@ ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/usr/share/grafana \ GRAFANA_VERSION="12.0.1" \ - JQ_VERSION="1.7.1" + JQ_VERSION="1.7.1" \ + CADDY_VERSION="2.9.1" \ + PYTHON_VERSION="3.13" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} @@ -15,6 +17,14 @@ COPY . / RUN install-packages fontconfig \ && install-stack grafana ${GRAFANA_VERSION} \ && install-stack jq ${JQ_VERSION} \ + && install-stack caddy $CADDY_VERSION \ + && install-stack python $PYTHON_VERSION && . init-stack \ + && python3 -m venv ${DRYCC_HOME_DIR}/venv \ + && source ${DRYCC_HOME_DIR}/venv/bin/activate \ + && pip3 install --disable-pip-version-check --no-cache-dir -r ${DRYCC_HOME_DIR}/oauth2/requirements.txt \ + && chown -R ${DRYCC_UID}:${DRYCC_GID} ${DRYCC_HOME_DIR} \ + # set env + && echo "source ${DRYCC_HOME_DIR}/venv/bin/activate" >> /opt/drycc/python/profile.d/python.sh \ && rm -rf \ /usr/share/doc \ /usr/share/man \ @@ -34,5 +44,4 @@ RUN install-packages fontconfig \ USER ${DRYCC_UID} WORKDIR ${DRYCC_HOME_DIR} -CMD ["start-grafana"] EXPOSE 3000 diff --git a/rootfs/usr/bin/start-grafana b/rootfs/usr/bin/start-grafana deleted file mode 100755 index 8d96a0d..0000000 --- a/rootfs/usr/bin/start-grafana +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -shopt -s expand_aliases - -alias curl="curl -s -H 'Content-Type: application/json' -H 'Accept: application/json'" -NAMESPACE=${NAMESPACE:-drycc} -GRAFANA_API_URL="http://${GF_SECURITY_ADMIN_USER}:${GF_SECURITY_ADMIN_PASSWORD}@localhost:${GF_SERVER_HTTP_PORT:-3000}" - -set -m -echo "Starting Grafana in the background" -exec grafana server --config /opt/drycc/grafana/conf/grafana.ini --homepath /opt/drycc/grafana & -echo "Waiting for Grafana to come up..." -until curl -q --fail --output /dev/null --silent "${GRAFANA_API_URL}/api/org"; do - printf "." - sleep 2 -done -echo "Grafana is up and running." - -# Create organization. if it doesn't exist -echo "Creating user organization..." -curl -XPOST "${GRAFANA_API_URL}/api/orgs" -d '{"name":"User Org."}' - -# Get the JSON for the all organization -GRAFANA_MAIN_ORG_ID=$(curl -XGET "${GRAFANA_API_URL}/api/orgs" | jq '.[] | select(.name == "Main Org.") | .id') -GRAFANA_USER_ORG_ID=$(curl -XGET "${GRAFANA_API_URL}/api/orgs" | jq '.[] | select(.name == "User Org.") | .id') - -# Prometheus Service Host and Port -echo "Using the following URL for prometheus: ${PROMETHEUS_URL}" -echo "Creating main organization prometheus datasource..." -curl -XPOST -H "X-Grafana-Org-Id: ${GRAFANA_MAIN_ORG_ID}" "${GRAFANA_API_URL}/api/datasources" -d ' -{ - "name": "prometheus-datasource", - "type": "prometheus", - "access": "proxy", - "isDefault": true, - "url": "'"${PROMETHEUS_URL}"'", - "jsonData": {"httpMethod": "GET", "oauthPassThru": true, "sigV4Auth": false} -}' -echo "Creating user organization prometheus datasource..." -curl -XPOST -H "X-Grafana-Org-Id: ${GRAFANA_USER_ORG_ID}" "${GRAFANA_API_URL}/api/datasources" -d ' -{ - "name": "prometheus-datasource", - "type": "prometheus", - "access": "proxy", - "isDefault": true, - "url": "'"${PROMETHEUS_URL}"'", - "jsonData": {"httpMethod": "GET", "oauthPassThru": true, "sigV4Auth": false} -}' - -echo "" -echo "Importing default dashboards..." -DASHBOARD_LOCATION="/usr/share/grafana/api/dashboards" -# replace namespace -# shellcheck disable=SC1003 -# shellcheck disable=SC2046 -sed -i 's@r\[\\"namespace\\\"\] == \\"drycc\\"@r\[\\"namespace\\"\] == \\"\'"${NAMESPACE}"'\\"@g' $(grep -F 'r[\"namespace\"] == \"drycc\"' -rl --include="*.json" "${DASHBOARD_LOCATION}/main") -# drycc component dashboard -DCD="${DASHBOARD_LOCATION}"/main/drycc_component_health.json - -# remove off-cluster component panel -if [ "on-cluster" != "${VALKEY_LOCATION}" ]; then - rm -rf "${DASHBOARD_LOCATION}"/main/drycc_valkey.json - # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "VALKEY"))')" > "${DCD}" -fi -if [ "on-cluster" != "${DATABASE_LOCATION}" ]; then - rm -rf "${DASHBOARD_LOCATION}"/main/drycc_database.json - # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "DATABASE"))')" > "${DCD}" -fi -if [ "on-cluster" != "${PASSPORT_LOCATION}" ]; then - # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "PASSPORT"))')" > "${DCD}" -fi -if [ "on-cluster" != "${REGISTRY_LOCATION}" ]; then - rm -rf "${DASHBOARD_LOCATION}"/main/drycc_registry.json - # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "REGISTRY"))')" > "${DCD}" - # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "REGISTRY-PROXY"))')" > "${DCD}" -fi -if [ "on-cluster" != "${STORAGE_LOCATION}" ]; then - rm -rf "${DASHBOARD_LOCATION}"/main/drycc_storage.json - # shellcheck disable=SC2005 - echo "$(<"${DCD}" jq 'del(.dashboard.panels[] | select(.title == "STORAGE"))')" > "${DCD}" -fi - -if [ "on-cluster" != "${PROMETHEUS_LOCATION}" ]; then - rm -rf "${DASHBOARD_LOCATION}"/main/drycc_victoriametrics_*.json -fi - -echo "Creating main organization grafana dashboard..." -for filename in "${DASHBOARD_LOCATION}"/main/*.json; do - echo "Importing ${filename} ..." - jq '{overwrite: true, dashboard: .}' "${filename}" | curl -XPOST --data-binary @- -H "X-Grafana-Org-Id: ${GRAFANA_MAIN_ORG_ID}" "${GRAFANA_API_URL}/api/dashboards/db" - echo "" - echo "Done importing ${filename}" -done - -echo "Creating main organization grafana dashboard..." -for filename in "${DASHBOARD_LOCATION}"/user/*.json; do - echo "Importing ${filename} ..." - jq '{overwrite: true, dashboard: .}' "${filename}" | curl -XPOST --data-binary @- -H "X-Grafana-Org-Id: ${GRAFANA_USER_ORG_ID}" "${GRAFANA_API_URL}/api/dashboards/db" - echo "" - echo "Done importing ${filename}" -done - -echo "Bringing Grafana back to the foreground" -fg diff --git a/rootfs/usr/share/grafana/api/dashboards/user/user_container.json b/rootfs/usr/share/grafana/api/dashboards/user/user_container.json deleted file mode 100644 index daa1104..0000000 --- a/rootfs/usr/share/grafana/api/dashboards/user/user_container.json +++ /dev/null @@ -1,2770 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 33, - "panels": [], - "targets": [ - { - "refId": "A" - } - ], - "title": "SUM", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 1 - }, - "hiddenSeries": false, - "id": 31, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_namespace", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",container!=\"\",namespace=\"drycc\"}[1m])) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["namespace"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total CPU Usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 1 - }, - "hiddenSeries": false, - "id": 32, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_namespace", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",container!=\"\",namespace=\"drycc\"})", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["namespace"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Total Memory Usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 52, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 2 - }, - "hiddenSeries": false, - "id": 54, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",container=~\"^drycc-passport.*$\"}[1m])) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "hide": false, - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 2 - }, - "hiddenSeries": false, - "id": 56, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-passport.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "PASSPORT", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 34, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 3 - }, - "hiddenSeries": false, - "id": 1, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-controller.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "hide": false, - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 3 - }, - "hiddenSeries": false, - "id": 2, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-controller.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "CONTROLLER", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 35, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 4 - }, - "hiddenSeries": false, - "id": 3, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-builder.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 4 - }, - "hiddenSeries": false, - "id": 4, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-builder.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "BUILDER", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 36, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 12 - }, - "hiddenSeries": false, - "id": 6, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-database.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 12 - }, - "hiddenSeries": false, - "id": 5, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-database.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "DATABASE", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 37, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 13 - }, - "hiddenSeries": false, - "id": 19, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-logger-fluentbit.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 13 - }, - "hiddenSeries": false, - "id": 20, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-logger-fluentbit.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "FLUENTBIT", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 38, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 7 - }, - "hiddenSeries": false, - "id": 25, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "datasource": null, - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-grafana.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 7 - }, - "hiddenSeries": false, - "id": 26, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "datasource": null, - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-grafana.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "GRAFANA", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 40, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 16 - }, - "hiddenSeries": false, - "id": 7, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=\"drycc-logger\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 16 - }, - "hiddenSeries": false, - "id": 8, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=\"drycc-logger\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "LOGGER", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 41, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 17 - }, - "hiddenSeries": false, - "id": 9, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 17 - }, - "hiddenSeries": false, - "id": 10, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-storage.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "STORAGE", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 43, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 21 - }, - "hiddenSeries": false, - "id": 27, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-valkey.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 21 - }, - "hiddenSeries": false, - "id": 28, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-valkey.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "VALKEY", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 44, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 22 - }, - "hiddenSeries": false, - "id": 11, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-registry.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 22 - }, - "hiddenSeries": false, - "id": 12, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=\"drycc-registry\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "REGISTRY", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 45, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 23 - }, - "hiddenSeries": false, - "id": 16, - "interval": "", - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-registry-proxy.*$\"}[1m])) by (pod) * 1000", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Millicores", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 23 - }, - "hiddenSeries": false, - "id": 15, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_pod_name", - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{image!=\"\",pod!=\"\",namespace=\"drycc\",container=~\"^drycc-registry-proxy.*$\"}) by (pod)", - "groupBy": [ - { - "params": ["$interval"], - "type": "time" - }, - { - "params": ["pod_name"], - "type": "tag" - }, - { - "params": ["null"], - "type": "fill" - } - ], - "legendFormat": "__auto", - "orderByTime": "ASC", - "policy": "default", - "range": true, - "refId": "A", - "resultFormat": "time_series" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "refId": "A" - } - ], - "title": "REGISTRY-PROXY", - "type": "row" - } - ], - "refresh": "5s", - "schemaVersion": 37, - "style": "dark", - "tags": ["drycc"], - "templating": { - "list": [] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] - }, - "timezone": "browser", - "title": "Drycc Component Health", - "uid": "2ySBrTTMz", - "version": 1, - "weekStart": "" -} diff --git a/rootfs/usr/share/grafana/api/dashboards/main/drycc_component_health.json b/rootfs/usr/share/grafana/dashboards/drycc_component_health.json similarity index 98% rename from rootfs/usr/share/grafana/api/dashboards/main/drycc_component_health.json rename to rootfs/usr/share/grafana/dashboards/drycc_component_health.json index daa1104..34db95a 100644 --- a/rootfs/usr/share/grafana/api/dashboards/main/drycc_component_health.json +++ b/rootfs/usr/share/grafana/dashboards/drycc_component_health.json @@ -21,7 +21,7 @@ } ] }, - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, @@ -51,7 +51,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -155,7 +155,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -267,7 +267,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -372,7 +372,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -494,7 +494,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -599,7 +599,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -721,7 +721,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -825,7 +825,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -947,7 +947,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -1051,7 +1051,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -1172,7 +1172,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -1276,7 +1276,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -1398,7 +1398,7 @@ "dashLength": 10, "dashes": false, "datasource": null, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -1504,7 +1504,7 @@ "dashLength": 10, "dashes": false, "datasource": null, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -1626,7 +1626,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -1730,7 +1730,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -1851,7 +1851,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -1955,7 +1955,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -2076,7 +2076,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -2179,7 +2179,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -2300,7 +2300,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -2404,7 +2404,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -2525,7 +2525,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, @@ -2629,7 +2629,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, diff --git a/rootfs/usr/share/grafana/api/dashboards/main/drycc_database.json b/rootfs/usr/share/grafana/dashboards/drycc_database.json similarity index 99% rename from rootfs/usr/share/grafana/api/dashboards/main/drycc_database.json rename to rootfs/usr/share/grafana/dashboards/drycc_database.json index 0ecb189..db2083b 100644 --- a/rootfs/usr/share/grafana/api/dashboards/main/drycc_database.json +++ b/rootfs/usr/share/grafana/dashboards/drycc_database.json @@ -13,7 +13,7 @@ ] }, "description": "A quickstart to setup the Prometheus PostgreSQL Exporter with preconfigured dashboards, alerting rules, and recording rules.", - "editable": true, + "editable": false, "gnetId": 14114, "graphTooltip": 0, "id": 1, @@ -25,7 +25,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fieldConfig": { "defaults": { @@ -356,7 +356,7 @@ "rgba(50, 172, 45, 0.97)" ], "decimals": 0, - "editable": true, + "editable": false, "error": false, "fieldConfig": { "defaults": { @@ -481,7 +481,7 @@ "dashLength": 10, "dashes": false, "decimals": 1, - "editable": true, + "editable": false, "error": false, "fieldConfig": { "defaults": { @@ -808,7 +808,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fieldConfig": { "defaults": { @@ -994,7 +994,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fieldConfig": { "defaults": { @@ -1096,7 +1096,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "editable": true, + "editable": false, "error": false, "fieldConfig": { "defaults": { diff --git a/rootfs/usr/share/grafana/api/dashboards/main/drycc_registry.json b/rootfs/usr/share/grafana/dashboards/drycc_registry.json similarity index 99% rename from rootfs/usr/share/grafana/api/dashboards/main/drycc_registry.json rename to rootfs/usr/share/grafana/dashboards/drycc_registry.json index fd6a364..d858dc1 100644 --- a/rootfs/usr/share/grafana/api/dashboards/main/drycc_registry.json +++ b/rootfs/usr/share/grafana/dashboards/drycc_registry.json @@ -12,7 +12,7 @@ } ] }, - "editable": true, + "editable": false, "gnetId": 9621, "graphTooltip": 0, "id": null, diff --git a/rootfs/usr/share/grafana/api/dashboards/main/drycc_storage.json b/rootfs/usr/share/grafana/dashboards/drycc_storage.json similarity index 99% rename from rootfs/usr/share/grafana/api/dashboards/main/drycc_storage.json rename to rootfs/usr/share/grafana/dashboards/drycc_storage.json index abb55b9..4926192 100644 --- a/rootfs/usr/share/grafana/api/dashboards/main/drycc_storage.json +++ b/rootfs/usr/share/grafana/dashboards/drycc_storage.json @@ -22,7 +22,7 @@ ] }, "description": "MinIO Grafana Dashboard - https://min.io/", - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "gnetId": 13502, "graphTooltip": 0, diff --git a/rootfs/usr/share/grafana/api/dashboards/main/drycc_valkey.json b/rootfs/usr/share/grafana/dashboards/drycc_valkey.json similarity index 99% rename from rootfs/usr/share/grafana/api/dashboards/main/drycc_valkey.json rename to rootfs/usr/share/grafana/dashboards/drycc_valkey.json index 043342c..07b0982 100644 --- a/rootfs/usr/share/grafana/api/dashboards/main/drycc_valkey.json +++ b/rootfs/usr/share/grafana/dashboards/drycc_valkey.json @@ -13,7 +13,7 @@ ] }, "description": "Redis Dashboard for Prometheus Redis Exporter 1.x", - "editable": true, + "editable": false, "gnetId": 763, "graphTooltip": 1, "id": null, @@ -41,7 +41,7 @@ "dashes": false, "datasource": "$datasource", "description": "Average taken across instances", - "editable": true, + "editable": false, "error": false, "fieldConfig": { "defaults": { @@ -148,7 +148,7 @@ "dashes": false, "datasource": "$datasource", "description": "Average taken across instances", - "editable": true, + "editable": false, "error": false, "fieldConfig": { "defaults": { @@ -258,7 +258,7 @@ "datasource": "$datasource", "decimals": 2, "description": "Hit rate shows the percentage of key space lookups that hit a key.", - "editable": true, + "editable": false, "error": false, "fieldConfig": { "defaults": { @@ -396,7 +396,7 @@ "dashes": false, "datasource": "$datasource", "description": "Total taken across instances", - "editable": true, + "editable": false, "error": false, "fieldConfig": { "defaults": { @@ -529,7 +529,7 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", - "editable": true, + "editable": false, "error": false, "fieldConfig": { "defaults": { @@ -645,7 +645,7 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", - "editable": true, + "editable": false, "error": false, "fieldConfig": { "defaults": { @@ -876,7 +876,7 @@ "dashes": false, "datasource": "$datasource", "description": "Sum taken across instances", - "editable": true, + "editable": false, "error": false, "fieldConfig": { "defaults": { @@ -985,7 +985,7 @@ "dashes": false, "datasource": "$datasource", "description": "Sum taken across instances", - "editable": true, + "editable": false, "error": false, "fieldConfig": { "defaults": { diff --git a/rootfs/usr/share/grafana/api/dashboards/main/drycc_victoriametrics_cluster.json b/rootfs/usr/share/grafana/dashboards/drycc_victoriametrics_cluster.json similarity index 99% rename from rootfs/usr/share/grafana/api/dashboards/main/drycc_victoriametrics_cluster.json rename to rootfs/usr/share/grafana/dashboards/drycc_victoriametrics_cluster.json index 629c4b4..0f35028 100644 --- a/rootfs/usr/share/grafana/api/dashboards/main/drycc_victoriametrics_cluster.json +++ b/rootfs/usr/share/grafana/dashboards/drycc_victoriametrics_cluster.json @@ -59,7 +59,7 @@ ] }, "description": "Overview for cluster VictoriaMetrics v1.102.0 or higher", - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": 1, diff --git a/rootfs/usr/share/grafana/api/dashboards/main/drycc_victoriametrics_vmagent.json b/rootfs/usr/share/grafana/dashboards/drycc_victoriametrics_vmagent.json similarity index 99% rename from rootfs/usr/share/grafana/api/dashboards/main/drycc_victoriametrics_vmagent.json rename to rootfs/usr/share/grafana/dashboards/drycc_victoriametrics_vmagent.json index d542fd0..46a825b 100644 --- a/rootfs/usr/share/grafana/api/dashboards/main/drycc_victoriametrics_vmagent.json +++ b/rootfs/usr/share/grafana/dashboards/drycc_victoriametrics_vmagent.json @@ -86,7 +86,7 @@ ] }, "description": "Overview for VictoriaMetrics vmagent v1.102.0 or higher", - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, diff --git a/rootfs/usr/share/grafana/dashboards/drycc_victoriametrics_vmauth.json b/rootfs/usr/share/grafana/dashboards/drycc_victoriametrics_vmauth.json new file mode 100644 index 0000000..f26e534 --- /dev/null +++ b/rootfs/usr/share/grafana/dashboards/drycc_victoriametrics_vmauth.json @@ -0,0 +1,2576 @@ +{ + "__inputs": [], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.4.2" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Overview for VictoriaMetrics vmauth v1.80.0 or higher", + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "doc", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "vmauth docs", + "tooltip": "vmauth docs", + "type": "link", + "url": "https://docs.victoriametrics.com/vmauth.html" + }, + { + "asDropdown": false, + "icon": "question", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Found a bug?", + "tooltip": "Found a bug?", + "type": "link", + "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues" + }, + { + "asDropdown": false, + "icon": "info", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "New releases", + "tooltip": "New releases", + "type": "link", + "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "Stats", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 11, + "x": 0, + "y": 1 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(min_over_time(vm_app_uptime_seconds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "Uptime", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows if the last configuration update was successful. \"Not Ok\" means there was an unsuccessful attempt to update the configuration due to some error. Check the log for details.", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "Ok" + } + }, + "type": "value" + }, + { + "options": { + "from": 1, + "result": { + "color": "red", + "index": 1, + "text": "Not Ok" + }, + "to": 999999 + }, + "type": "range" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 11, + "y": 1 + }, + "id": 30, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": false, + "expr": "count(vmauth_config_last_reload_successful{job=~\"$job\", instance=~\"$instance\"} < 1 ) or 0", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Config update", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 7, + "x": 17, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(sum(rate(vmauth_user_requests_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) or 0) + (sum(rate(vmauth_unauthorized_user_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) or 0)", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Requests rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the total number of users defined at configuration file.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 11, + "y": 4 + }, + "id": 31, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(vmauth_user_concurrent_requests_capacity{job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Users count", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of request errors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 7, + "x": 17, + "y": 4 + }, + "id": 36, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(vmauth_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Errors rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false, + "minWidth": 50 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Count" + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 11, + "x": 0, + "y": 5 + }, + "id": 6, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by (job, short_version)", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Version", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 13, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmauth_user_requests_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by(username)", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmauth_unauthorized_user_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Requests rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows percent utilization of per concurrent requests capacity.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "mappings": [], + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(\nmax_over_time(vmauth_user_concurrent_requests_current{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])\n/ \nvmauth_user_concurrent_requests_capacity{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}\n) by(username) > 0\n", + "hide": false, + "interval": "5m", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "User concurrent requests usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the rate of rejected requests by a reason.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmauth_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (reason)", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Requests rejected rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": " The number of concurrent connections processed by vmauth reached one of limits. Possible solutions:\n- increase global limit with flag -maxConcurrentRequests\n- increase limit with flag: -maxConcurrentPerUserRequests for all users or with config option `max_concurrent_requests` per user.\n- deploy additional vmauth replicas\n- check requests latency at backend service and allocate resources to it if needed", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmauth_user_concurrent_requests_limit_reached_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by(username) > 0", + "interval": "1m", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmauth_unauthorized_user_concurrent_requests_limit_reached_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0", + "hide": false, + "legendFormat": "unauthorized", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vmauth_concurrent_requests_limit_reached_total[$__rate_interval])) > 0", + "hide": false, + "legendFormat": "global at {{ $instance }}", + "range": true, + "refId": "C" + } + ], + "title": "Concurrent limit reached", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows duration in seconds of user requests by quantile.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(vmauth_user_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", username=~\"$user\", quantile=~\"(0.99|0.5)\"}) by (quantile, username) > 0", + "hide": false, + "legendFormat": "user: {{username}} q: {{ quantile}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(vmauth_unauthorized_user_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", quantile=~\"(0.99|0.5)\"}) by (quantile) > 0", + "hide": false, + "legendFormat": "user: unauthorized q: {{ quantile}}", + "range": true, + "refId": "B" + } + ], + "title": "User requests duration", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 22, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Percentage of used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RSS memory % usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU % usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 27, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "requested from system", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "heap inuse", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(go_memstats_stack_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "stack inuse", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "resident", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": false, + "expr": "sum(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "resident anonymous", + "refId": "E" + } + ], + "title": "Memory usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU cores used", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": false, + "expr": "process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Limit", + "refId": "B" + } + ], + "title": "CPU ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "connections", + "refId": "A" + } + ], + "title": "TCP connections ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "connections", + "range": true, + "refId": "A" + } + ], + "title": "TCP connections rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percentage of open file descriptors compared to the limit set in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max_over_time(process_open_fds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n/\nprocess_max_fds{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Open FDs ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(go_goroutines{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "gc duration", + "refId": "A" + } + ], + "title": "Goroutines ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(process_num_threads{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "threads", + "refId": "A" + } + ], + "title": "Threads ($instance)", + "type": "timeseries" + } + ], + "title": "Resource usage", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 35, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 34, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "job" + } + ] + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(flag{is_set=\"true\", job=~\"$job\", instance=~\"$instance\"}) by(job, instance, name, value)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Non-default flags", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "instance": { + "aggregations": [] + }, + "job": { + "aggregations": [] + }, + "name": { + "aggregations": [], + "operation": "groupby" + }, + "value": { + "aggregations": [], + "operation": "groupby" + } + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows number of generated error and warning messages in logs. Non-zero value may be a sign of connectivity or missconfiguration errors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "max", + "last", + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(increase(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (level)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Log errors", + "type": "timeseries" + } + ], + "title": "Troubleshooting", + "type": "row" + } + ], + "refresh": "30s", + "revision": 1, + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "VictoriaMetrics - cluster", + "value": "PAF93674D0B4E9963" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "ds", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{version=~\"^vmauth.*\"}, job)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(vm_app_version{version=~\"^vmauth.*\"}, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vm_app_version{job=~\"$job\"}, instance)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(vm_app_version{job=~\"$job\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "definition": "label_values(vmauth_user_requests_total{job=~\"$job\", instance=~\"$instance\"}, username)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "user", + "options": [], + "query": { + "query": "label_values(vmauth_user_requests_total{job=~\"$job\", instance=~\"$instance\"}, username)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "P38648FE0F8C5BEA2" + }, + "filters": [], + "hide": 0, + "name": "adhoc", + "skipUrlSync": false, + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "VictoriaMetrics - vmauth", + "uid": "nbuo5Mr4k", + "version": 1, + "weekStart": "", + "gnetId": 21394 +} \ No newline at end of file diff --git a/rootfs/usr/share/grafana/api/dashboards/main/kubernetes.json b/rootfs/usr/share/grafana/dashboards/kubernetes.json similarity index 99% rename from rootfs/usr/share/grafana/api/dashboards/main/kubernetes.json rename to rootfs/usr/share/grafana/dashboards/kubernetes.json index 34566cf..af96df7 100644 --- a/rootfs/usr/share/grafana/api/dashboards/main/kubernetes.json +++ b/rootfs/usr/share/grafana/dashboards/kubernetes.json @@ -23,7 +23,7 @@ ] }, "description": "【English version】2025.01.25 Updates，kubernetesComprehensive display of resources！Includes K8S Overall Resource Overview、Microservices Resource Details、Pod Resource Details andK8SNetwork Bandwidth，Optimization metrics。https://grafana.com/orgs/starsliao/dashboards", - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, diff --git a/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_apiserver.json b/rootfs/usr/share/grafana/dashboards/kubernetes_apiserver.json similarity index 99% rename from rootfs/usr/share/grafana/api/dashboards/main/kubernetes_apiserver.json rename to rootfs/usr/share/grafana/dashboards/kubernetes_apiserver.json index 9fda701..9862cf6 100644 --- a/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_apiserver.json +++ b/rootfs/usr/share/grafana/dashboards/kubernetes_apiserver.json @@ -12,7 +12,7 @@ } ] }, - "editable": true, + "editable": false, "gnetId": 12006, "graphTooltip": 0, "id": null, diff --git a/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_cluster.json b/rootfs/usr/share/grafana/dashboards/kubernetes_cluster.json similarity index 98% rename from rootfs/usr/share/grafana/api/dashboards/main/kubernetes_cluster.json rename to rootfs/usr/share/grafana/dashboards/kubernetes_cluster.json index 1d47310..8a0b173 100644 --- a/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_cluster.json +++ b/rootfs/usr/share/grafana/dashboards/kubernetes_cluster.json @@ -5,20 +5,20 @@ "tags": ["kubernetes"], "style": "dark", "timezone": "browser", - "editable": true, + "editable": false, "hideControls": false, "sharedCrosshair": false, "rows": [ { "collapse": false, - "editable": true, + "editable": false, "height": "200px", "panels": [ { "aliasColors": {}, "bars": false, "decimals": 2, - "editable": true, + "editable": false, "error": false, "fill": 1, "grid": { @@ -115,7 +115,7 @@ }, { "collapse": false, - "editable": true, + "editable": false, "height": "250px", "panels": [ { @@ -127,7 +127,7 @@ "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "editable": true, + "editable": false, "error": false, "format": "percent", "gauge": { @@ -207,7 +207,7 @@ "rgba(245, 54, 54, 0.9)" ], "decimals": 2, - "editable": true, + "editable": false, "error": false, "format": "percent", "gauge": { @@ -286,7 +286,7 @@ "rgba(245, 54, 54, 0.9)" ], "decimals": 2, - "editable": true, + "editable": false, "error": false, "format": "percent", "gauge": { @@ -367,7 +367,7 @@ "rgba(245, 54, 54, 0.9)" ], "decimals": 2, - "editable": true, + "editable": false, "error": false, "format": "bytes", "gauge": { @@ -446,7 +446,7 @@ "rgba(245, 54, 54, 0.9)" ], "decimals": 2, - "editable": true, + "editable": false, "error": false, "format": "bytes", "gauge": { @@ -525,7 +525,7 @@ "rgba(245, 54, 54, 0.9)" ], "decimals": 2, - "editable": true, + "editable": false, "error": false, "format": "none", "gauge": { @@ -604,7 +604,7 @@ "rgba(245, 54, 54, 0.9)" ], "decimals": 2, - "editable": true, + "editable": false, "error": false, "format": "none", "gauge": { @@ -683,7 +683,7 @@ "rgba(245, 54, 54, 0.9)" ], "decimals": 2, - "editable": true, + "editable": false, "error": false, "format": "bytes", "gauge": { @@ -762,7 +762,7 @@ "rgba(245, 54, 54, 0.9)" ], "decimals": 2, - "editable": true, + "editable": false, "error": false, "format": "bytes", "gauge": { @@ -837,14 +837,14 @@ }, { "collapse": false, - "editable": true, + "editable": false, "height": "250px", "panels": [ { "aliasColors": {}, "bars": false, "decimals": 3, - "editable": true, + "editable": false, "error": false, "fill": 0, "grid": { @@ -931,14 +931,14 @@ }, { "collapse": true, - "editable": true, + "editable": false, "height": "250px", "panels": [ { "aliasColors": {}, "bars": false, "decimals": 3, - "editable": true, + "editable": false, "error": false, "fill": 0, "grid": { @@ -1024,14 +1024,14 @@ }, { "collapse": true, - "editable": true, + "editable": false, "height": "250px", "panels": [ { "aliasColors": {}, "bars": false, "decimals": 3, - "editable": true, + "editable": false, "error": false, "fill": 0, "grid": { @@ -1139,14 +1139,14 @@ }, { "collapse": true, - "editable": true, + "editable": false, "height": "500px", "panels": [ { "aliasColors": {}, "bars": false, "decimals": 3, - "editable": true, + "editable": false, "error": false, "fill": 0, "grid": { @@ -1233,14 +1233,14 @@ }, { "collapse": false, - "editable": true, + "editable": false, "height": "250px", "panels": [ { "aliasColors": {}, "bars": false, "decimals": 2, - "editable": true, + "editable": false, "error": false, "fill": 0, "grid": { @@ -1325,14 +1325,14 @@ }, { "collapse": true, - "editable": true, + "editable": false, "height": "250px", "panels": [ { "aliasColors": {}, "bars": false, "decimals": 2, - "editable": true, + "editable": false, "error": false, "fill": 0, "grid": { @@ -1417,14 +1417,14 @@ }, { "collapse": true, - "editable": true, + "editable": false, "height": "250px", "panels": [ { "aliasColors": {}, "bars": false, "decimals": 2, - "editable": true, + "editable": false, "error": false, "fill": 0, "grid": { @@ -1527,14 +1527,14 @@ }, { "collapse": true, - "editable": true, + "editable": false, "height": "500px", "panels": [ { "aliasColors": {}, "bars": false, "decimals": 2, - "editable": true, + "editable": false, "error": false, "fill": 0, "grid": { @@ -1619,14 +1619,14 @@ }, { "collapse": false, - "editable": true, + "editable": false, "height": "250px", "panels": [ { "aliasColors": {}, "bars": false, "decimals": 2, - "editable": true, + "editable": false, "error": false, "fill": 1, "grid": { @@ -1720,14 +1720,14 @@ }, { "collapse": true, - "editable": true, + "editable": false, "height": "250px", "panels": [ { "aliasColors": {}, "bars": false, "decimals": 2, - "editable": true, + "editable": false, "error": false, "fill": 1, "grid": { @@ -1863,14 +1863,14 @@ }, { "collapse": true, - "editable": true, + "editable": false, "height": "500px", "panels": [ { "aliasColors": {}, "bars": false, "decimals": 2, - "editable": true, + "editable": false, "error": false, "fill": 1, "grid": { diff --git a/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_deployment_metrics.json b/rootfs/usr/share/grafana/dashboards/kubernetes_deployment_metrics.json similarity index 99% rename from rootfs/usr/share/grafana/api/dashboards/main/kubernetes_deployment_metrics.json rename to rootfs/usr/share/grafana/dashboards/kubernetes_deployment_metrics.json index 231f591..cc9be62 100644 --- a/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_deployment_metrics.json +++ b/rootfs/usr/share/grafana/dashboards/kubernetes_deployment_metrics.json @@ -22,7 +22,7 @@ ] }, "description": "Monitors Kubernetes deployments in cluster using Prometheus. Shows overall cluster CPU / Memory / Network of deployments, replicas in each deployment. Uses Kube state metrics and cAdvisor metrics ", - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "gnetId": 741, "graphTooltip": 0, @@ -688,7 +688,7 @@ "dashes": false, "datasource": null, "decimals": 3, - "editable": true, + "editable": false, "error": false, "fill": 0, "fillGradient": 0, @@ -787,7 +787,7 @@ "dashes": false, "datasource": null, "decimals": 3, - "editable": true, + "editable": false, "error": false, "fill": 0, "fillGradient": 0, @@ -889,7 +889,7 @@ "dashes": false, "datasource": null, "decimals": 2, - "editable": true, + "editable": false, "error": false, "fill": 0, "fillGradient": 0, @@ -987,7 +987,7 @@ "dashes": false, "datasource": {}, "decimals": 2, - "editable": true, + "editable": false, "error": false, "fill": 0, "fillGradient": 0, @@ -1085,7 +1085,7 @@ "dashes": false, "datasource": null, "decimals": 2, - "editable": true, + "editable": false, "error": false, "fill": 1, "fillGradient": 0, diff --git a/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_metrics.json b/rootfs/usr/share/grafana/dashboards/kubernetes_metrics.json similarity index 99% rename from rootfs/usr/share/grafana/api/dashboards/main/kubernetes_metrics.json rename to rootfs/usr/share/grafana/dashboards/kubernetes_metrics.json index d3dae70..5e07a52 100644 --- a/rootfs/usr/share/grafana/api/dashboards/main/kubernetes_metrics.json +++ b/rootfs/usr/share/grafana/dashboards/kubernetes_metrics.json @@ -16,7 +16,7 @@ ] }, "description": "Summary metrics about kube-state-metrics v2 version(https://github.com/kubernetes/kube-state-metrics); Referenced 6417", - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, diff --git a/rootfs/usr/share/grafana/api/dashboards/main/node.json b/rootfs/usr/share/grafana/dashboards/node.json similarity index 99% rename from rootfs/usr/share/grafana/api/dashboards/main/node.json rename to rootfs/usr/share/grafana/dashboards/node.json index e4e067d..f76826d 100644 --- a/rootfs/usr/share/grafana/api/dashboards/main/node.json +++ b/rootfs/usr/share/grafana/dashboards/node.json @@ -22,7 +22,7 @@ ] }, "description": "cpu memroy disk IO network temperature", - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "gnetId": 8919, "graphTooltip": 0, @@ -64,7 +64,7 @@ "type": "prometheus" }, "description": "", - "editable": true, + "editable": false, "error": false, "gridPos": { "h": 1, diff --git a/rootfs/usr/share/grafana/oauth2/__init__.py b/rootfs/usr/share/grafana/oauth2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rootfs/usr/share/grafana/oauth2/alerting/test.json b/rootfs/usr/share/grafana/oauth2/alerting/test.json new file mode 100644 index 0000000..0708db4 --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/alerting/test.json @@ -0,0 +1,74 @@ +{ + "title": "TEST-API_1", + "ruleGroup": "API", + "folderUID": "SET_FOLDER_UID", + "noDataState": "OK", + "execErrState": "OK", + "for": "5m", + "orgId": 1, + "uid": "", + "condition": "B", + "annotations": { + "summary": "test_api_1" + }, + "labels": { + "API": "test1" + }, + "data": [ + { + "refId": "A", + "queryType": "", + "relativeTimeRange": { + "from": 600, + "to": 0 + }, + "datasourceUid": "XXXXXXXXX-XXXXXXXXX-XXXXXXXXXX", + "model": { + "expr": "up", + "hide": false, + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": "A" + } + }, + { + "refId": "B", + "queryType": "", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "-100", + "model": { + "conditions": [ + { + "evaluator": { + "params": [6], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": ["A"] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "datasource": { + "type": "__expr__", + "uid": "-100" + }, + "hide": false, + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": "B", + "type": "classic_conditions" + } + } + ] +} \ No newline at end of file diff --git a/rootfs/usr/share/grafana/oauth2/dashboard/node.json b/rootfs/usr/share/grafana/oauth2/dashboard/node.json new file mode 100644 index 0000000..f76826d --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/dashboard/node.json @@ -0,0 +1,3613 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "cpu memroy disk IO network temperature", + "editable": false, + "fiscalYearStartMonth": 0, + "gnetId": 8919, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "More", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 176, + "panels": [], + "title": "Basic info", + "type": "row" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "editable": false, + "error": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 11, + "links": [], + "maxPerRow": 6, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "", + "mode": "html" + }, + "pluginVersion": "9.5.3", + "repeat": "node", + "repeatDirection": "h", + "style": {}, + "title": "$node", + "type": "text" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 2 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 0, + "y": 2 + }, + "hideTimeOverride": true, + "id": 15, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "time() - node_boot_time_seconds{instance=~\"$node\"}", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 2, + "refId": "A", + "step": 40 + } + ], + "title": "uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 2 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 2, + "y": 2 + }, + "id": 14, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "expr": "count(count(node_cpu_seconds_total{instance=~\"$node\", mode='system'}) by (cpu))", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 20 + } + ], + "title": "cpu cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 2 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 3 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 4, + "y": 2 + }, + "id": 75, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "title": "memory", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 6, + "y": 2 + }, + "hideTimeOverride": false, + "id": 171, + "links": [], + "maxDataPoints": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum(node_filesystem_size_bytes {instance=~\"$node\",fstype=~\"ext4|xfs\"})", + "format": "table", + "hide": false, + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "refId": "A" + } + ], + "title": "system disk", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 50 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 8, + "y": 2 + }, + "id": 167, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "expr": "100 - (avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"idle\"}[5m])) * 100)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 20 + } + ], + "title": "cpu usage（5m）", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 20 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 50 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 11, + "y": 2 + }, + "id": 20, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"iowait\"}[5m])) * 100", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 20 + } + ], + "title": "CPU iowait（5m）", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 14, + "y": 2 + }, + "hideTimeOverride": false, + "id": 172, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "expr": "(1 - (node_memory_MemAvailable_bytes{instance=~\"$node\"} / (node_memory_MemTotal_bytes{instance=~\"$node\"})))* 100", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 20 + } + ], + "title": "Memory Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "Obtain the maximum partition through the variable maxmount.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 17, + "y": 2 + }, + "id": 154, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.5.3", + "repeatDirection": "h", + "targets": [ + { + "expr": "100 - ((node_filesystem_avail_bytes{instance=~\"$node\",mountpoint=\"$maxmount\",fstype=~\"ext4|xfs\"} * 100) / node_filesystem_size_bytes {instance=~\"$node\",mountpoint=\"$maxmount\",fstype=~\"ext4|xfs\"})", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 20 + } + ], + "title": "partion ($maxmount) usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "max": 10000, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 7000 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 9000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 21, + "y": 2 + }, + "hideTimeOverride": false, + "id": 16, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "expr": "node_filefd_allocated{instance=~\"$node\"}", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "refId": "B" + } + ], + "title": "opend file descriptor", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "15 minutes" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "1 minute" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5分钟" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 0, + "y": 7 + }, + "id": 13, + "links": [], + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "expr": "node_load1{instance=~\"$node\"}", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 2, + "legendFormat": "{{instance}}_1m", + "metric": "", + "refId": "A", + "step": 20, + "target": "" + }, + { + "expr": "node_load5{instance=~\"$node\"}", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 2, + "legendFormat": "{{instance}}_5m", + "refId": "B", + "step": 20 + }, + { + "expr": "node_load15{instance=~\"$node\"}", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 2, + "legendFormat": "{{instance}}_15m", + "refId": "C", + "step": 20 + } + ], + "title": "Load average", + "type": "timeseries" + }, + { + "aliasColors": { + "Idle - Waiting for something to happen": "#052B51", + "guest": "#9AC48A", + "idle": "#052B51", + "iowait": "#EAB839", + "irq": "#BF1B00", + "nice": "#C15C17", + "sdb_I/O operations per second%": "#d683ce", + "softirq": "#E24D42", + "steal": "#FCE2DE", + "system": "#508642", + "user": "#5195CE", + "The proportion of disk spent on I/O operations": "#ba43a9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "decimals": 2, + "description": "node_disk_io_time_seconds_total", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 13, + "x": 11, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"system\"}[2m])) by (instance)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}_System", + "refId": "A", + "step": 20 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"user\"}[2m])) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}_User", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"idle\"}[2m])) by (instance)", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{instance}}_Idle", + "range": true, + "refId": "F", + "step": 240 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"iowait\"}[2m])) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}_Iowait", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}_{{device}}_per second I/O%", + "range": true, + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU usage、disk I/O time（%）", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:513", + "format": "percentunit", + "label": "", + "logBase": 1, + "max": "1", + "show": true + }, + { + "$$hashKey": "object:514", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "displayName": "", + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 11, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 207, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"}-node_filesystem_avail_bytes {instance=~'$node',fstype=~\"ext4|xfs\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}_usage", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Partion usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3570", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:3571", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "mountpoint" + }, + "properties": [ + { + "id": "displayName", + "value": "partion" + }, + { + "id": "unit", + "value": "bytes" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)" + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "free" + }, + { + "id": "unit", + "value": "bytes" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10000000000 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 20000000000 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "usage" + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "displayName", + "value": "total" + }, + { + "id": "unit", + "value": "bytes" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "fstype" + }, + "properties": [ + { + "id": "displayName", + "value": "file system" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "displayName", + "value": "file system" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "device" + }, + "properties": [ + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 13, + "x": 11, + "y": 14 + }, + "id": 164, + "links": [], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "file system" + } + ] + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "exemplar": false, + "expr": "node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "C" + }, + { + "exemplar": true, + "expr": "node_filesystem_avail_bytes {instance=~'$node',fstype=~\"ext4|xfs\"}", + "format": "table", + "hide": true, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + }, + { + "exemplar": false, + "expr": "1-(node_filesystem_free_bytes{instance=~'$node',fstype=~\"ext4|xfs\"} / node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "B" + } + ], + "title": "Disk usage", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "fstype", + "mountpoint", + "Value #C", + "Value #A", + "Value #B", + "device" + ] + } + } + } + ], + "type": "table" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 211, + "panels": [ + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "MEM_Avaliable" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "MEM_Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "MEM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "MEM_Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6d1f62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "MEM_Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#eab839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "avaliable" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9ac48a", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "totol memery" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#bf1b00", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 2 + }, + "id": 156, + "links": [], + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}_TotalMem", + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"} - node_memory_MemAvailable_bytes{instance=~\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}_Used", + "range": true, + "refId": "B", + "step": 4 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_MemFree_bytes{instance=~\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "MEM_Free", + "range": true, + "refId": "C", + "step": 4 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_Buffers_bytes{instance=~\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "MEM_Buffers", + "range": true, + "refId": "D", + "step": 4 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_Cached_bytes{instance=~\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "MEM_Cached", + "range": true, + "refId": "E", + "step": 4 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_MemAvailable_bytes{instance=~\"$node\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}_Available", + "range": true, + "refId": "F", + "step": 4 + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"} - (node_memory_Cached_bytes{instance=~\"$node\"} + node_memory_Buffers_bytes{instance=~\"$node\"} + node_memory_MemFree_bytes{instance=~\"$node\"})", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "refId": "G" + } + ], + "title": "Memory info", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 2 + }, + "id": 206, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "exemplar": true, + "expr": "node_memory_SwapFree_bytes{instance=~\"$node\"}", + "hide": false, + "interval": "", + "legendFormat": "Swap_Free", + "refId": "A" + }, + { + "exemplar": true, + "expr": "node_memory_SwapCached_bytes{instance=~\"$node\"}", + "hide": false, + "interval": "", + "legendFormat": "SwapCached", + "refId": "B" + }, + { + "exemplar": true, + "expr": "node_memory_SwapTotal_bytes{instance=~\"$node\"}", + "hide": false, + "interval": "", + "legendFormat": "SwapTotal", + "refId": "C" + } + ], + "title": "Swap memory", + "type": "timeseries" + } + ], + "title": "Memory", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 209, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 3 + }, + "height": "300", + "hiddenSeries": false, + "id": 157, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:101", + "alias": "/.*_out transmit$/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_network_receive_bytes_total{instance=~'$node',device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*|lxc.*|nodelocaldns|cilium.*'}[5m])*8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}_{{device}}_in receive", + "range": true, + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_network_transmit_bytes_total{instance=~'$node',device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*|lxc.*|nodelocaldns|cilium.*'}[5m])*8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}_{{device}}_out transmit", + "range": true, + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network flow", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:85", + "format": "bps", + "label": "upload（-）/download（+）", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:86", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "TCP": "#6ED0E0" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 3 + }, + "height": "300", + "hiddenSeries": false, + "id": 158, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_netstat_Tcp_CurrEstab{instance=~'$node'}", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{instance}}_ESTABLISHED", + "refId": "A", + "step": 20 + }, + { + "expr": "node_sockstat_TCP_tw{instance=~'$node'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}_TCP_tw", + "refId": "D" + }, + { + "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=~'$node'}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}_ActiveOpens", + "refId": "B" + }, + { + "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=~'$node'}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}_PassiveOpens", + "refId": "C" + }, + { + "expr": "node_sockstat_TCP_alloc{instance=~'$node'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}_TCP_alloc", + "refId": "E" + }, + { + "expr": "node_sockstat_TCP_inuse{instance=~'$node'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}_TCP_inuse", + "refId": "F" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TCP connect info", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:269", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:270", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Network", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 190, + "panels": [ + { + "aliasColors": { + "vda_write": "#6ED0E0" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "Reads completed: Reads completed per second for each disk partition\n\nWrites completed: \nIO now", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 24 + }, + "height": "300", + "hiddenSeries": false, + "id": 161, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:386", + "alias": "/.*_read$/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_reads_completed_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}_{{device}}_read", + "range": true, + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_writes_completed_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}_{{device}}_write", + "range": true, + "refId": "B", + "step": 10 + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "node_disk_io_now{instance=~\"$node\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "disk read and write（IOPS）", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:393", + "format": "iops", + "label": "read（-）/write（+）I/O ops/sec", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:394", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "vda": "#6ED0E0" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "Read time ms The number of seconds taken for each disk partition read operation\n\nWrite time ms\nIO time ms \nIO time weighted", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 24 + }, + "height": "300", + "hiddenSeries": false, + "id": 160, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:459", + "alias": "/,*_read$/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "range": true, + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}}_weighting", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_read_time_seconds_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}_{{device}}_read", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_write_time_seconds_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}_{{device}}_write", + "range": true, + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "disk io read write time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:466", + "format": "s", + "label": "read（-）/write（+）", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:467", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "The number of read operations per second=[Δrd_ios/Δt]", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 194, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideZero": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_disk_reads_completed_total{instance=~'$node'}[2m])", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "r/s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1217", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1218", + "format": "bytes", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "The number of write operations per second=[Δwr_ios/Δt]", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "hiddenSeries": false, + "id": 196, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "(rate(node_disk_writes_completed_total{instance=~\"$node\"}[2m]))", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "w/s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1363", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1364", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "The average time required for each read operation=[ Δ Rd_ticks/ Δ Rd_ios not only includes the time required for hard disk device read operations, but also the waiting time in the kernel queue.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "hiddenSeries": false, + "id": 202, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "(rate(node_disk_read_time_seconds_total {instance=~'$node'}[2m])) /(rate(node_disk_reads_completed_total {instance=~'$node'}[2m]))", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "r_await", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2148", + "format": "s", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2149", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "Average time required for each write operation=[ Δ Wr_ticks/ Δ Wr_ios not only includes the time required for hard disk device write operations, but also the waiting time in the kernel queue.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "hiddenSeries": false, + "id": 204, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "(rate(node_disk_write_time_seconds_total {instance=~'$node'}[2m])) /(rate(node_disk_writes_completed_total {instance=~'$node'}[2m]))", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "w_await", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2477", + "format": "s", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2478", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 198, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "(rate(node_disk_read_bytes_total {instance=~'$node'}[2m]))", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "rMiB/s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1622", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1623", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "hiddenSeries": false, + "id": 200, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "(rate(node_disk_written_bytes_total {instance=~'$node'}[2m]))", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "wKiB/s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1920", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1921", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "vda_write": "#6ED0E0" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 55 + }, + "height": "300", + "hiddenSeries": false, + "id": 168, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*_read$/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_read_bytes_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}_{{device}}_read", + "range": true, + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_written_bytes_total{instance=~\"$node\"}[2m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}_{{device}}_write", + "range": true, + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk read and write capacity size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": "read（-）/write（+）", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "description": "The busy ratio of this hard drive device=[ Δ Io_ticks/ Δ t] The time ratio indicating that the device has I/O (i.e. non idle), regardless of how much I/O there is, only considering whether there is", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 65 + }, + "hiddenSeries": false, + "id": 192, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideZero": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "idc", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[2m])", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "%util", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:744", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:745", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Disk read write", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 213, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 169, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.5.3", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "editorMode": "code", + "expr": "node_hwmon_temp_celsius{instance=~'$node'}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{chip}}_{{sensor}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cpu temperature", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3976", + "format": "celsius", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:3977", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Cpu temperature", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": ["instance"], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 10, + "auto_min": "10s", + "current": { + "selected": true, + "text": "5s", + "value": "5s" + }, + "hide": 0, + "label": "interval", + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": true, + "text": "5s", + "value": "5s" + }, + { + "selected": false, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "allFormat": "glob", + "allValue": "", + "current": { + "selected": false, + "text": [], + "value": [] + }, + "datasource": { + "type": "prometheus" + }, + "definition": "label_values(node_exporter_build_info{},instance)", + "hide": 0, + "includeAll": false, + "label": "host instance", + "multi": true, + "multiFormat": "regex values", + "name": "node", + "options": [], + "query": { + "query": "label_values(node_exporter_build_info{},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "/", + "value": "/" + }, + "datasource": { + "type": "prometheus" + }, + "definition": "", + "hide": 2, + "includeAll": false, + "label": "", + "multi": false, + "name": "maxmount", + "options": [], + "query": { + "query": "query_result(topk(1,sort_desc (max(node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"}) by (mountpoint))))", + "refId": "Prometheus-maxmount-Variable-Query" + }, + "refresh": 2, + "regex": "/.*\\\"(.*)\\\".*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "datasource": {}, + "filters": [], + "hide": 0, + "label": "", + "name": "Filters", + "skipUrlSync": false, + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "browser", + "title": "Node Metrics", + "uid": "SQlD2vTZq", + "version": 1, + "weekStart": "" +} diff --git a/rootfs/usr/share/grafana/oauth2/hook/__init__.py b/rootfs/usr/share/grafana/oauth2/hook/__init__.py new file mode 100644 index 0000000..e8dd6f7 --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/hook/__init__.py @@ -0,0 +1,7 @@ +from .grafana import ( + init_org, sync_user, sync_role, sync_datasource, sync_alerting, sync_dashboard +) + +startup_hooks = [init_org] +login_hooks = [sync_user, sync_role, sync_datasource, sync_alerting, sync_dashboard] +destroy_hooks = [] diff --git a/rootfs/usr/share/grafana/oauth2/hook/__pycache__/__init__.cpython-312.pyc b/rootfs/usr/share/grafana/oauth2/hook/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6609f3de5f3920434aa49e7daf97795f7ffd481c GIT binary patch literal 327 zcmXv}Jxjzu5Zz5&Zxe~fS@;V?h@klcVsCTJAwrt4-j17K+zq=)k;aeCPCNYzhX|Jb z2OCRSECee%;W{ffAx<&#=FOYo&1)2e$i?yFEWdU2t84y)3}CtKz&+xK;}Si27y%YI z^}vS}Ag~G}K~gGE8nj;Qe`6<(vBf*nIJ6_7*(sCE(n1z>rnIo6L&gTCmGi7Ipa-Z` z33lAGoYl;zM$f@UJ;6%Qbs>daALb@MQA~47a;46Ur5sGHRcjV#*_r;`O^)1UlkkSg zTuIRu$*ez^u&PSBr6kLGOW9u4TDpI(cvHfGp1AGq1GDAQkB>3_K%1YzP94mkf&pthw123izL%UALCRKP${p!XkB9bm<{ z0)21zX<4?D7X70u&Fq^uv$Hd^^WOWtH~h!EJOcyQfBjwWiQiQ)%)enFeF-UI$*y6T zD~!m9WSHqs-bp{9Wlg_^-r0Uu)QD`5^m74bi1wN8CV3yqd%TP#krTCFLf?4x3nD+1 zpQLm1AJCidB}qQ{1D?^PpUI3v4^J27jvz@7l|0FNO3Tvk-Lia@mgdZeQnL~DnXy?! zL%&`u5sg13{Z`Qw$UmkL^M1!{%;uX9? z;FYZ46#>tV{lj~Ufi4{<&eA{PB)T4%JfV?rU?kk!H$ED8Wc81Z4hIMQ;b3UQH6({d z)FN-s5sznJZ;#vO?s593mp;jkbdb+#%db$rB z==QKIk-P0FgTUuO^s#M3KG(I@k($?D2*0LG0ZJSy~ zf}<_L&|qr!(eHpX3u&ngvMWrKNLrCZnFy468Bvp_oNQMlnCkb)%k9!ZPq*jLfv&y1 z;kH9ZezdpuaBq0anb4Un$A`8Y*?v+!c6OWW-SpbaM_T=-&h~YV_`46g{o%1*fA45p zkM9LKvgYHG|CoQozsd+B4Y^ZzMnfm3^2b8bDSueigk)6+jGPWiufK-j5Kbz{=- zz^FfbLe&N30l#=EI3nYO9gauqU5Dg=B)h~h|Hz5Z$no)@t2ZA>Ksu8)Jc+Zx#sWP9c7qBeqqReSV z$`7Z62c7^Pwn&Q=^rWXxYI+%`p-aM-O89owHYx><1tcjT4vfKqjQCFlBF3J;sZco3 zj4PBWbU}@(4V>}^ha;v`gH9}|d`LPz5ELI-Uu)i*{G`p@((&f~qLZks9G0A{RDiG2 z;R}Kz$3m)bBH$Nc^<^A~!{LyuSZ~BdPSr>iz^R2KzXjQOCT^)zs@fFuhVy&lm1~vy z?Xk)o=l4#&9OKHKD6grXIul#tg6ggqR~{EEm)c`O*Y3XlZMPFh_&bP^~@ldW9L1VnWOx~DH%)4MJ&cXD4rMbfX zOsi!k*eE-sO;zXI!fAuUh-{Ski1`G*`HhT$h-z-|X+`)h7$Mpto9dB^azpx@u2N)r z3=`U@Rut$yAtxK*d)xGA#^5j0Bbc$=F%o8+fl(Tt?DKh}6on`Wk{h}&@q@y(a;)Tg zyt$pZ_#vcIkbIEJA?4LFZOpqkO4zHvqVsNNw2Vy7Xb5uzz6*}{EPI4GLw?PkAx?eH z&W;#Bl7lYTo3g9#3nE!ySa3t@qjI>-ztE%#sHvU2szsf1RyDfi@sUB#a1hkGg!Y@t zyi|kFv7=*QCoiE@lpL7R-uOYF zIl)4CIbndRM%9KxCj%oA8nwaHo&=I%bdacI&txR~|4^byTK|SKavBA#o|eBG z8EU1xOrUJ9B8QL3cG5-wpiF9ZQH!!J?E(p21$O30Fb~#&!?qtWqVNeT>TYDf*6QOF z4PZZL?;zvG|D}w70&@K$NTraBKTy6+T@gK%YP2L$6-+_Z1VvTAB3K~R1Xf-i8x2Uv zh~(3z)atvCA-f_b7gU}K$}%7{xldYyJ(z=X%51q2AV}4~bP7@}+@w0pvPuAr64eCK zT5QBqyC$teF77l5L4ni&naa@@u2chhR4vrbN^Q_;2PFBQAftNV8NzRzcWnCmlXn}G z-q+_l-ayfRbHVZ^h`y!ZgYoyre|`?g%Ic}QYyFeLC4Ed+7B?1L>bhsF1?L&8>Y01T3sPLU2zd+#1rM^>?pM9QAjunG{P&c z9L1FP{5L~yW*nzz5v?ChyugEkUIvyP9mXhe5lS0FRq|(!5k&RRT zYz3(DVqiEB4yfE{2q8Yaz^QwNI>h0qc_;iSFN{JVz@&_f2AKk-3|2}iNjj=sP`M9V z1z<6rqa>Bat@Dm;b35*yQC@p{-s49td33>Y^l>d|ubj;LLqW-;CvLG_YJWd^^+)$C zj<}=la^8cwhH2r`1DAxWx_j2@gq|t1e`{b&rBjX59rtRs-#l?Ea<8KY6>9V4W~xxv zPamE+5^LVAtbakVyYG7MTVKg)(}b2Oa4az_Do{4h{4LFNjIC;F-}GDe>UQ4J-7UNK zLLc_mbh#<1pl8DOtvj<_#OqrWmnT;5y?Z=X^eSrPH7hl8RT7>3sgW@jTy24Q=-d-1K?`A06X(!$`{?1l63x(Mo#JiE7-N^wx#~9&(IV16I=H^Ug zNY|526F1kuBkdyCrj6zsX}*~tzn$;Y3v;_jCu5rH5P<$#s{{UH${l?tiriiWBJGo9 z)jJ5W28B3{xWJD_k1H4ZK`mZk^n2-*Zi`|~MUw53>`+6+0K8W0Pvyr(ML%qGi?`F~ zK6IdOU|*-(+u76WeP4movWC^^>j-cmIy7{f>5cp~3n1VxS(Si*Xbtm|P6dUIYg`!G9)^wUim0LlyOwEPh_ow$fXxEoPn*6LEVf?o6)Ih zHx}&MkSW_|ni6?x?Y@B8Rjft^2u>Xc6Vhrx|<;1gUJ!s9Rm34(?G!NGtmtNNpUIXH+fk-X4ELNo6)srFUF;D>q+ zJQa?pMgT?OF?nDx1R+HlrYReM@Gn4&fk7z%s(XNjn+K8+0d(A??UGHOS(52-6@CM?uUWIK^C=ox?{KS~3i zxVcy^sST{{_37#=o7mVdGWBFD2p0OrgT6w+nTIomSlP|`roY#MR?Z?wY zH*4mrb}dx7V^!{3&bxi{RXvKSV$rc(+40idL-UT_g|gnsm*SS+W4m3xA@qm zy_XDEx5upYQ{ypf+k?XT>H3-Sn}=g{H zURm>?YTfjfSk;Dj!Fm?vn zLj+#Q*w}Xo4J`C`2@_M^N0t*N81HhT91&Qeo+c~&Wq+3c0rtR!CZCJ^ zn)r15-?Bh`wa!<@-YzFTBY(TXT?mCct;A>M@3gtCP?#gcXXWQKTBHRdJTPY?J{vbz zphenlL4E`A74mb9ZUYp)-avfC{MQ>fq}%aTjl- z)7(xuWxQ{_@Qs7`YE9o%3rIH#(DE@5hi9Y|xBMdOp#v*&H}a5 ziYv-SGbk$E?`CF#4`;B&YG=4nKHU~039FsxicA<$FTF(e;|%^F3ce0hy>d|>#3-~P4}Q9kUh53{m3+#9`+VR}ufQ!G$c-cSH z#_PHwnwAz-FZ$&ZM?-!|bZVsaFeO!h6EX0s!JzrCG0|Uo`!-0W;tWi^i;h-Fb1}YaCg}^nnW884a zFG{^mObAe_J(YhDHxPsew{;_$q49$T2$9Ra|a!%KoG#E}epRz;l{YTG@mbC14WBlA?)udAo5uN?9XI=K?U*m>KHv3(0a~<|T{m1a zOm+N=eZ%Cw#r&cVk6t-``S_LL%fnM0Gh642cizw6bt`|d!f}27wf$56j}BbY$E~Fc zR!7Y0nC9+p>xxx(-?tu&7gt<4dHE#Tg^Jes!nUj_c(Ar5zSjAzmMtqrlTfq72xt-n zGzpMB6c}5FlD`WMElM{&#FLApHU<|MYM0a1kH8-0`oe#fQ8q8Q%R9B?b|vwy;cu^D zk*;=YysZ$rbeDT;gxL<_br@!Mb3o6r2B?_JGeWC5JMp$~bEQ1e4uZ8cT5pqau0g;y zEd=?ke5Xd3+d{nCOmpo5(%|-L$?AuY_)CM%IzVN7UpG!V2~c|AjS!EvK> z4qW6_G)))-*YpAplB36xEF48AUgabjj&_j z?Ax{5KOFSSyHdZH$nk`d%21b8bjj0{Lof&l135`WXP`b!rzC|Ws^pwn=?$!;Co5=y zvRGTj?MlHjh0fApti|jzNODmIE?d@2HB7IaF~`>JR-7*=4!2_7bAIm=I_mev@_1?0 z^|EVaiv>-JZT<91%GP~K`+mjxqSE&|D2}qaC5Du}Nggt!@J%x5ePAhAuvEn?RZ|=9 zS?c2@-_;&2z+e)`zDLlHw zYDz#!X+Tr)4CuGlFTcKMX;93KO5;Jr^a>uz=A%w4MwnZS2eMWJ;B*6HEli&HJWPfD z?!m!qg2S;bC)HS@5SlJ0idLw!_h5N0)Oro%uZX9K|7#YgFKRrS+1p0qY36U6DBWTL z_U-M&1ChQRTA=SRZY?yL)e=t|Kg$;&U5brn*Afo|{pxw7n+ftExCe9_r6ITn^k&{G z2oN6hbeLv$3P8`X0`Q&02Uc?v3`o^d*T`8$jiuxRR`PhQ7O597!^lu_&)|ob5I{Vitf5exY+zMq4W#!&1rPzq|6fd-v_>cK3McS-#4N!ND-Kp)_z!&*CJEHPZkc zb-t(xr2Fa%_~8Ur(VwX3cAg4}W5a_1D%f+v@F6Y-P88i5pCEd(1$3YH3u#E_^k5cAc(1OvC}I(oYayB$i2vLez$ zunO;3hj*;Rd!kiC>ZS@|_maA4k=|~K;7*ddgrSKTrZy!QxXp0%c8goWQsNC_CblW) b32xJ2db=53!cw9ppOoJe|HQzJF7=0.68.0 +redis[asyncio]>=6.2.0 +uvicorn[standard]>=0.15.0 +authlib>=1.0.0 +httpx>=0.23.0 +python-jose>=3.3.0 +python-multipart>=0.0.5 +itsdangerous>=2.2.0 diff --git a/rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml b/rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml new file mode 100644 index 0000000..eecb3a0 --- /dev/null +++ b/rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml @@ -0,0 +1 @@ +apiVersion: 1 \ No newline at end of file diff --git a/rootfs/usr/share/grafana/provisioning/dashboards/drycc.yaml b/rootfs/usr/share/grafana/provisioning/dashboards/drycc.yaml new file mode 100644 index 0000000..b9ca478 --- /dev/null +++ b/rootfs/usr/share/grafana/provisioning/dashboards/drycc.yaml @@ -0,0 +1,7 @@ +apiVersion: 1 +providers: +- name: 'default' + orgId: 1 + type: file + options: + path: /usr/share/grafana/dashboards diff --git a/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml b/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml new file mode 100644 index 0000000..404a459 --- /dev/null +++ b/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml @@ -0,0 +1,14 @@ +apiVersion: 1 +datasources: +- name: Prometheus on Drycc + type: prometheus + access: proxy + orgId: 1 + url: $DRYCC_VICTORIAMETRICS_URL + basicAuth: true + basicAuthUser: $DRYCC_VICTORIAMETRICS_USERNAME + isDefault: true + secureJsonData: + basicAuthPassword: $DRYCC_VICTORIAMETRICS_PASSWORD + version: 1 + editable: false \ No newline at end of file diff --git a/rootfs/usr/share/grafana/provisioning/plugins/drycc.yaml b/rootfs/usr/share/grafana/provisioning/plugins/drycc.yaml new file mode 100644 index 0000000..55439cb --- /dev/null +++ b/rootfs/usr/share/grafana/provisioning/plugins/drycc.yaml @@ -0,0 +1 @@ +apiVersion: 1 From ab29100c98f1fcf3612e07af407d6119e8e008ac Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 25 Jun 2025 10:58:08 +0800 Subject: [PATCH 25/50] chore(grafana): remove vmauth --- charts/grafana/templates/_helpers.tmpl | 18 ++++-------------- .../grafana/templates/grafana-deployment.yaml | 6 +----- .../provisioning/datasources/drycc.yaml | 8 ++++---- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/charts/grafana/templates/_helpers.tmpl b/charts/grafana/templates/_helpers.tmpl index 96c5370..fee868f 100644 --- a/charts/grafana/templates/_helpers.tmpl +++ b/charts/grafana/templates/_helpers.tmpl @@ -21,23 +21,13 @@ env: - name: DRYCC_VALKEY_URL value: "redis://:$(DRYCC_VALKEY_PASSWORD)@drycc-valkey.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:16379/0" {{- end }} -{{- if .Values.victoriametricsUrl }} -- name: "DRYCC_VICTORIAMETRICS_URL" - value: "{{ .Values.victoriametricsUrl }}" -{{- else if .Values.victoriametrics.enabled }} -- name: "DRYCC_VICTORIAMETRICS_USERNAME" - valueFrom: - secretKeyRef: - name: victoriametrics-vmauth-creds - key: username -- name: "DRYCC_VICTORIAMETRICS_PASSWORD" +- name: DRYCC_SERVICE_KEY valueFrom: secretKeyRef: - name: victoriametrics-vmauth-creds - key: password + name: controller-creds + key: service-key - name: "DRYCC_VICTORIAMETRICS_URL" - value: "http://$(DRYCC_VICTORIAMETRICS_USERNAME):$(DRYCC_VICTORIAMETRICS_PASSWORD)@drycc-victoriametrics-vmauth.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:8427/select/0/prometheus" -{{- end }} + value: "http://$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)/v2/prometheus/drycc" {{- if .Values.passport.enabled}} - name: "DRYCC_PASSPORT_URL" {{- if .Values.global.certManagerEnabled }} diff --git a/charts/grafana/templates/grafana-deployment.yaml b/charts/grafana/templates/grafana-deployment.yaml index c71566c..b77cb9b 100644 --- a/charts/grafana/templates/grafana-deployment.yaml +++ b/charts/grafana/templates/grafana-deployment.yaml @@ -57,11 +57,7 @@ spec: DRYCC_VALKEY_JSON=$(echo "$DRYCC_VALKEY_URL" | jq -R 'capture("^(?redis[s]?)://(?[^:]+)?(?::(?[^@]+))?@(?
[^/]+)/?(?[0-9]*)$")') GF_LIVE_HA_ENGINE_ADDRESS=$(echo "${DRYCC_VALKEY_JSON}" |jq -r '.address') GF_LIVE_HA_ENGINE_PASSWORD=$(echo "${DRYCC_VALKEY_JSON}" |jq -r '.password') - DRYCC_VICTORIAMETRICS_JSON=$(echo "$DRYCC_VICTORIAMETRICS_URL" | jq -R 'capture("^(?https?://)(?[^:@]+):(?[^@]+)@(?
.+)$") | {url: "\(.scheme)\(.address)", username: .username, password: .password}') - DRYCC_VICTORIAMETRICS_URL=$(echo "${DRYCC_VICTORIAMETRICS_JSON}" |jq -r '.url') - DRYCC_VICTORIAMETRICS_USERNAME=$(echo "${DRYCC_VICTORIAMETRICS_JSON}" |jq -r '.username') - DRYCC_VICTORIAMETRICS_PASSWORD=$(echo "${DRYCC_VICTORIAMETRICS_JSON}" |jq -r '.password') - export GF_LIVE_HA_ENGINE_ADDRESS GF_LIVE_HA_ENGINE_PASSWORD DRYCC_VICTORIAMETRICS_URL DRYCC_VICTORIAMETRICS_USERNAME DRYCC_VICTORIAMETRICS_PASSWORD + export GF_LIVE_HA_ENGINE_ADDRESS GF_LIVE_HA_ENGINE_PASSWORD exec grafana server --config /usr/share/grafana/grafana.ini --homepath /opt/drycc/grafana {{- end }} {{- with index .Values "resources" }} diff --git a/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml b/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml index 404a459..5079bfa 100644 --- a/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml +++ b/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml @@ -5,10 +5,10 @@ datasources: access: proxy orgId: 1 url: $DRYCC_VICTORIAMETRICS_URL - basicAuth: true - basicAuthUser: $DRYCC_VICTORIAMETRICS_USERNAME isDefault: true + jsonData: + httpHeaderName1: X-Drycc-Service-Key secureJsonData: - basicAuthPassword: $DRYCC_VICTORIAMETRICS_PASSWORD + httpHeaderValue1: $DRYCC_SERVICE_KEY version: 1 - editable: false \ No newline at end of file + editable: false From a95a8fa8250e9afe8708103f7bbf45ce7bc15263 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 11 Jul 2025 09:25:40 +0800 Subject: [PATCH 26/50] chore(grafana): check grafana running --- charts/grafana/templates/grafana-deployment.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/charts/grafana/templates/grafana-deployment.yaml b/charts/grafana/templates/grafana-deployment.yaml index b77cb9b..f3d0570 100644 --- a/charts/grafana/templates/grafana-deployment.yaml +++ b/charts/grafana/templates/grafana-deployment.yaml @@ -120,6 +120,12 @@ spec: - -ec - | set -m + echo "Waiting for Grafana to come up..." + until curl -q --fail --output /dev/null --silent "http://${GF_SECURITY_ADMIN_USER}:${GF_SECURITY_ADMIN_PASSWORD}@localhost:${GF_SERVER_HTTP_PORT:-3000}/api/org"; do + printf "." + sleep 2 + done + echo "Grafana is up and running." python3 oauth2/main.py \ --port 4000 \ --client-id $(DRYCC_PASSPORT_KEY) \ From d2228dc0202a6f5f18d7347613d7a17be54d4acc Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 11 Jul 2025 16:38:01 +0800 Subject: [PATCH 27/50] chore(quickwit): add grafana quickwit plugin --- rootfs/Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 8be4e5d..9b111c0 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -14,7 +14,7 @@ RUN groupadd drycc --gid ${DRYCC_GID} \ COPY . / -RUN install-packages fontconfig \ +RUN install-packages unzip fontconfig \ && install-stack grafana ${GRAFANA_VERSION} \ && install-stack jq ${JQ_VERSION} \ && install-stack caddy $CADDY_VERSION \ @@ -25,6 +25,11 @@ RUN install-packages fontconfig \ && chown -R ${DRYCC_UID}:${DRYCC_GID} ${DRYCC_HOME_DIR} \ # set env && echo "source ${DRYCC_HOME_DIR}/venv/bin/activate" >> /opt/drycc/python/profile.d/python.sh \ + # grafana plugin + && mkdir ${DRYCC_HOME_DIR}/plugins \ + && curl -sSL https://github.com/quickwit-oss/quickwit-datasource/releases/download/v0.4.6/quickwit-quickwit-datasource-0.4.6.zip -o quickwit.zip \ + && unzip quickwit.zip -d ${DRYCC_HOME_DIR}/plugins \ + && rm -rf quickwit.zip \ && rm -rf \ /usr/share/doc \ /usr/share/man \ From 3866753784538af36bc7ac39417467be2a4c3d2f Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 11 Jul 2025 17:16:27 +0800 Subject: [PATCH 28/50] chore(prometheus): add quickwit --- .../grafana/dashboards/drycc_quickwit.json | 1286 +++++++++++++++++ 1 file changed, 1286 insertions(+) create mode 100644 rootfs/usr/share/grafana/dashboards/drycc_quickwit.json diff --git a/rootfs/usr/share/grafana/dashboards/drycc_quickwit.json b/rootfs/usr/share/grafana/dashboards/drycc_quickwit.json new file mode 100644 index 0000000..545013f --- /dev/null +++ b/rootfs/usr/share/grafana/dashboards/drycc_quickwit.json @@ -0,0 +1,1286 @@ +{ + "__inputs": [ + { + "name": "datasource", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Dashboard to monitor Quickwit indexers.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.1", + "targets": [ + { + "editorMode": "builder", + "expr": "sum by(docs_processed_status) (rate(quickwit_indexing_processed_bytes{index=~\"$index\", namespace=\"$namespace\", instance=~\"$instance\", pod=~\"$pod\"}[$quickwit_rate_interval]))", + "legendFormat": "{{docs_processed_status}}", + "range": true, + "refId": "A" + } + ], + "title": "Indexing throughput", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "suffix: docs/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.1", + "targets": [ + { + "editorMode": "builder", + "expr": "sum by(docs_processed_status) (rate(quickwit_indexing_processed_docs_total{index=~\"$index\", namespace=\"$namespace\", pod=~\"$pod\"}[$quickwit_rate_interval]))", + "legendFormat": "{{docs_processed_status}}", + "range": true, + "refId": "A" + } + ], + "title": "Documents throughput", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "axisSoftMax": -4, + "axisSoftMin": 8, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "valid" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd", + "seriesBy": "last" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "parsing_error" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "continuous-RdYlGr" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(docs_processed_status, index) (rate(quickwit_indexing_processed_bytes{index=~\"$index\", namespace=\"$namespace\", pod=~\"$pod\", instance=~\"$instance\"}[$quickwit_rate_interval]))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{docs_processed_status}}-{{index}}", + "range": true, + "refId": "Indexing bytes rate" + } + ], + "title": "Indexing throughput", + "transformations": [], + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "editorMode": "builder", + "exemplar": false, + "expr": "rate(quickwit_write_bytes{index=~\"$index\", pod=~\"$pod\", namespace=\"$namespace\"}[$quickwit_rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{component}} {{index}} pod:{{pod}}", + "range": true, + "refId": "Writes" + } + ], + "title": "Writes rate", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.1", + "targets": [ + { + "editorMode": "builder", + "expr": "quickwit_allocated_num_bytes{namespace=\"$namespace\", pod=~\"$pod\"}", + "legendFormat": "allocated-{{pod}}", + "range": true, + "refId": "A" + }, + { + "editorMode": "builder", + "expr": "process_resident_memory_bytes{namespace=\"$namespace\", pod=~\"$pod\"}", + "hide": false, + "legendFormat": "rss-{{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "Memory consumption (allocated and RSS)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 14, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "editorMode": "builder", + "expr": "quickwit_indexing_ongoing_merge_operations{index=~\"$index\", namespace=\"$namespace\", instance=~\"$instance\", pod=~\"$pod\"}", + "hide": false, + "legendFormat": "{{index}} {{pod}}", + "range": true, + "refId": "Processed docs" + } + ], + "title": "Ongoing merge operations", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 7, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Upload bytes / sec" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "editorMode": "builder", + "expr": "sum by(pod) (rate(quickwit_storage_object_storage_download_num_bytes{namespace=\"$namespace\", instance=~\"$instance\", pod=~\"$pod\"}[$quickwit_rate_interval]))", + "legendFormat": "Download bytes / sec - {{pod}}", + "range": true, + "refId": "Download" + }, + { + "editorMode": "builder", + "expr": "sum by(pod) (rate(quickwit_storage_object_storage_upload_num_bytes{namespace=\"$namespace\", pod=~\"$pod\", instance=~\"$instance\"}[$__rate_interval]))", + "hide": false, + "legendFormat": "Upload bytes / sec - {{pod}}", + "range": true, + "refId": "Upload" + } + ], + "title": "Object storage transfer rate", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 14, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "editorMode": "builder", + "expr": "sum by(docs_processed_status, index) (rate(quickwit_indexing_processed_docs_total{index=~\"$index\", namespace=\"$namespace\", instance=~\"$instance\", pod=~\"$pod\"}[$quickwit_rate_interval]))", + "hide": false, + "legendFormat": "{{docs_processed_status}}-{{index}}", + "range": true, + "refId": "Processed docs" + } + ], + "title": "Indexed documents rate", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 7, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Upload bytes / sec" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "editorMode": "builder", + "expr": "sum(rate(quickwit_storage_object_storage_gets_total{namespace=\"$namespace\", instance=~\"$instance\", pod=~\"$pod\"}[$quickwit_rate_interval]))", + "legendFormat": "GET req/sec", + "range": true, + "refId": "Download" + }, + { + "editorMode": "builder", + "expr": "sum(rate(quickwit_storage_object_storage_puts_total{namespace=\"$namespace\", pod=~\"$pod\", instance=~\"$instance\"}[$__rate_interval]))", + "hide": false, + "legendFormat": "PUT req/sec", + "range": true, + "refId": "Upload" + } + ], + "title": "Requests on object storage", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "editorMode": "builder", + "expr": "histogram_quantile(0.75, rate(quickwit_cli_thread_unpark_duration_microseconds_bucket{namespace=\"$namespace\", pod=~\"$pod\"}[5m]))", + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Thread unpark duration", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.1", + "targets": [ + { + "editorMode": "builder", + "expr": "rate(quickwit_indexing_backpressure_micros{namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])", + "legendFormat": "{{actor_name}}-{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Backpressure", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "quickwit", + "indexer" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "definition": "label_values(quickwit_write_bytes,namespace)", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(quickwit_write_bytes,namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "definition": "label_values(quickwit_write_bytes,index)", + "hide": 0, + "includeAll": true, + "label": "Index", + "multi": false, + "name": "index", + "options": [], + "query": { + "query": "label_values(quickwit_write_bytes,index)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "definition": "label_values(quickwit_write_bytes,pod)", + "hide": 0, + "includeAll": true, + "label": "Pod", + "multi": true, + "name": "pod", + "options": [], + "query": { + "query": "label_values(quickwit_write_bytes,pod)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "definition": "label_values(quickwit_write_bytes, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(quickwit_write_bytes, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "30s", + "value": "30s" + }, + "description": "", + "hide": 0, + "label": "Rate interval", + "name": "quickwit_rate_interval", + "options": [ + { + "selected": false, + "text": "10s", + "value": "10s" + }, + { + "selected": true, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Quickwit Indexers", + "uid": "quickwit-indexers", + "version": 50, + "weekStart": "", + "gnetId": 20214 +} \ No newline at end of file From 5dafcb0a453a691e199644ac0471c2057119b05a Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Sat, 12 Jul 2025 23:59:23 +0800 Subject: [PATCH 29/50] chore(grafana): add check oauth2 ok --- charts/grafana/templates/_helpers.tmpl | 2 ++ charts/grafana/templates/grafana-deployment.yaml | 10 ++++++++-- rootfs/Dockerfile | 2 +- .../share/grafana/dashboards/drycc_quickwit.json | 2 +- .../grafana/provisioning/datasources/drycc.yaml | 14 ++++++++++++++ 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/charts/grafana/templates/_helpers.tmpl b/charts/grafana/templates/_helpers.tmpl index fee868f..1c0599b 100644 --- a/charts/grafana/templates/_helpers.tmpl +++ b/charts/grafana/templates/_helpers.tmpl @@ -26,6 +26,8 @@ env: secretKeyRef: name: controller-creds key: service-key +- name: "DRYCC_QUICKWIT_URL" + value: "http://$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)/v2/quickwit/drycc" - name: "DRYCC_VICTORIAMETRICS_URL" value: "http://$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)/v2/prometheus/drycc" {{- if .Values.passport.enabled}} diff --git a/charts/grafana/templates/grafana-deployment.yaml b/charts/grafana/templates/grafana-deployment.yaml index f3d0570..93a4a45 100644 --- a/charts/grafana/templates/grafana-deployment.yaml +++ b/charts/grafana/templates/grafana-deployment.yaml @@ -144,7 +144,7 @@ spec: livenessProbe: tcpSocket: port: oauth2 - initialDelaySeconds: 120 + initialDelaySeconds: 240 periodSeconds: 10 timeoutSeconds: 5 failureThreshold: 6 @@ -181,6 +181,12 @@ spec: - -ec - | set -m + echo "Waiting for Grafana oauth2 proxy to come up..." + until curl -q --fail --output /dev/null --silent "http://localhost:4000/oauth2/healthz"; do + printf "." + sleep 2 + done + echo "Grafana oauth2 proxy is up and running." caddy run --config /usr/share/grafana/Caddyfile {{- end }} {{- with index .Values "resources" }} @@ -195,7 +201,7 @@ spec: livenessProbe: tcpSocket: port: proxy - initialDelaySeconds: 120 + initialDelaySeconds: 300 periodSeconds: 10 timeoutSeconds: 5 failureThreshold: 6 diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 9b111c0..21ce9a7 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -4,7 +4,7 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/usr/share/grafana \ - GRAFANA_VERSION="12.0.1" \ + GRAFANA_VERSION="12.0.2" \ JQ_VERSION="1.7.1" \ CADDY_VERSION="2.9.1" \ PYTHON_VERSION="3.13" diff --git a/rootfs/usr/share/grafana/dashboards/drycc_quickwit.json b/rootfs/usr/share/grafana/dashboards/drycc_quickwit.json index 545013f..e707c56 100644 --- a/rootfs/usr/share/grafana/dashboards/drycc_quickwit.json +++ b/rootfs/usr/share/grafana/dashboards/drycc_quickwit.json @@ -59,7 +59,7 @@ ] }, "description": "Dashboard to monitor Quickwit indexers.", - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, diff --git a/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml b/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml index 5079bfa..c64470a 100644 --- a/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml +++ b/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml @@ -1,5 +1,19 @@ apiVersion: 1 datasources: +- name: Application Logs + type: quickwit-quickwit-datasource + access: proxy + orgId: 1 + url: $DRYCC_QUICKWIT_URL + jsonData: + httpHeaderName1: X-Drycc-Service-Key + index: logs-* + logMessageField: log + secureJsonData: + httpHeaderValue1: $DRYCC_SERVICE_KEY + version: 1 + editable: false + - name: Prometheus on Drycc type: prometheus access: proxy From 71425413ce034b7be499a240f585d7082b07848d Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 15 Jul 2025 10:59:59 +0800 Subject: [PATCH 30/50] chore(grafana): set datasource read only --- .../usr/share/grafana/oauth2/hook/grafana.py | 18 ++++++++++++++++-- .../usr/share/grafana/oauth2/requirements.txt | 1 + 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/rootfs/usr/share/grafana/oauth2/hook/grafana.py b/rootfs/usr/share/grafana/oauth2/hook/grafana.py index 7a067cb..574f03d 100644 --- a/rootfs/usr/share/grafana/oauth2/hook/grafana.py +++ b/rootfs/usr/share/grafana/oauth2/hook/grafana.py @@ -1,6 +1,7 @@ import os import json import httpx +from psycopg import AsyncConnection DEFAULT_HEADERS = {"Content-Type": "application/json"} DRYCC_GRAFANA_REFRESH = os.environ.get('DRYCC_GRAFANA_REFRESH', '60s') @@ -112,14 +113,16 @@ async def sync_datasource(context: dict, token: dict, userinfo: dict): datasource["secureJsonData"] = { "httpHeaderValue1": f"Token {drycc_token["token"]}", } + await _set_datasource_read_only(False, datasource["id"]) await client.put( api_url(f"/api/datasources/uid/{datasource["uid"]}"), headers=api_headers(context, userinfo), json=datasource, ) + await _set_datasource_read_only(True, datasource["id"]) return drycc_token = await _get_or_create_drycc_token(None, token) - await client.post( + resp = await client.post( api_url("/api/datasources"), headers=api_headers(context, userinfo), json={ @@ -139,6 +142,7 @@ async def sync_datasource(context: dict, token: dict, userinfo: dict): }, }, ) + await _set_datasource_read_only(True, resp.json()["datasource"]["id"]) async def sync_dashboard(context: dict, token: dict, userinfo: dict): @@ -158,7 +162,17 @@ async def sync_dashboard(context: dict, token: dict, userinfo: dict): ) -async def _get_or_create_drycc_token(drycc_token_uuid, token: dict): +async def _set_datasource_read_only(read_only: bool, id: str): + async with await AsyncConnection.connect(os.environ.get("GF_DATABASE_URL")) as conn: + async with conn.cursor() as cursor: + await cursor.execute( + "UPDATE data_source SET read_only=%s WHERE id=%s", + (read_only, id) + ) + await conn.commit() + + +async def _get_or_create_drycc_token(drycc_token_uuid: str, token: dict): headers = {"Authorization": f"Bearer {token["access_token"]}"} async with httpx.AsyncClient() as client: if drycc_token_uuid: diff --git a/rootfs/usr/share/grafana/oauth2/requirements.txt b/rootfs/usr/share/grafana/oauth2/requirements.txt index 6b0938e..4061b5f 100644 --- a/rootfs/usr/share/grafana/oauth2/requirements.txt +++ b/rootfs/usr/share/grafana/oauth2/requirements.txt @@ -3,6 +3,7 @@ redis[asyncio]>=6.2.0 uvicorn[standard]>=0.15.0 authlib>=1.0.0 httpx>=0.23.0 +psycopg[binary]>=3.2.9 python-jose>=3.3.0 python-multipart>=0.0.5 itsdangerous>=2.2.0 From 3c64fd1dabaef3fb9ab0d2e04f483e3950a605c6 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 15 Jul 2025 16:48:17 +0800 Subject: [PATCH 31/50] chore(oauth2): add user datasources --- .../{dashboard => dashboards}/node.json | 0 .../oauth2/datasources/prometheus.json | 16 ++ .../grafana/oauth2/datasources/quickwit.json | 15 ++ .../usr/share/grafana/oauth2/hook/__init__.py | 4 +- .../usr/share/grafana/oauth2/hook/grafana.py | 141 +++++++++--------- 5 files changed, 105 insertions(+), 71 deletions(-) rename rootfs/usr/share/grafana/oauth2/{dashboard => dashboards}/node.json (100%) create mode 100644 rootfs/usr/share/grafana/oauth2/datasources/prometheus.json create mode 100644 rootfs/usr/share/grafana/oauth2/datasources/quickwit.json diff --git a/rootfs/usr/share/grafana/oauth2/dashboard/node.json b/rootfs/usr/share/grafana/oauth2/dashboards/node.json similarity index 100% rename from rootfs/usr/share/grafana/oauth2/dashboard/node.json rename to rootfs/usr/share/grafana/oauth2/dashboards/node.json diff --git a/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json b/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json new file mode 100644 index 0000000..450d976 --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json @@ -0,0 +1,16 @@ +{ + "name": "Prometheus on Drycc", + "type": "prometheus", + "url": "${controller_api_url}/v2/prometheus/${username}", + "access": "proxy", + "isDefault": true, + "basicAuth": false, + "jsonData": { + "httpHeaderName1": "Authorization", + "httpMethod": "GET", + "timeInterval": "${time_interval}" + }, + "secureJsonData": { + "httpHeaderValue1": "Token ${token}" + } +} \ No newline at end of file diff --git a/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json b/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json new file mode 100644 index 0000000..99e7fff --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json @@ -0,0 +1,15 @@ +{ + "name": "Application Logs", + "type": "quickwit-quickwit-datasource", + "url": "${controller_api_url}/v2/quickwit/${username}", + "access": "proxy", + "basicAuth": false, + "jsonData": { + "httpHeaderName1": "Authorization", + "index": "logs-*", + "logMessageField": "log" + }, + "secureJsonData": { + "httpHeaderValue1": "Token ${token}" + } +} diff --git a/rootfs/usr/share/grafana/oauth2/hook/__init__.py b/rootfs/usr/share/grafana/oauth2/hook/__init__.py index e8dd6f7..4651a65 100644 --- a/rootfs/usr/share/grafana/oauth2/hook/__init__.py +++ b/rootfs/usr/share/grafana/oauth2/hook/__init__.py @@ -1,7 +1,7 @@ from .grafana import ( - init_org, sync_user, sync_role, sync_datasource, sync_alerting, sync_dashboard + init_org, sync_user, sync_role, sync_datasources, sync_alerting, sync_dashboards ) startup_hooks = [init_org] -login_hooks = [sync_user, sync_role, sync_datasource, sync_alerting, sync_dashboard] +login_hooks = [sync_user, sync_role, sync_datasources, sync_alerting, sync_dashboards] destroy_hooks = [] diff --git a/rootfs/usr/share/grafana/oauth2/hook/grafana.py b/rootfs/usr/share/grafana/oauth2/hook/grafana.py index 574f03d..f3b7e9d 100644 --- a/rootfs/usr/share/grafana/oauth2/hook/grafana.py +++ b/rootfs/usr/share/grafana/oauth2/hook/grafana.py @@ -1,6 +1,7 @@ import os import json import httpx +from string import Template from psycopg import AsyncConnection DEFAULT_HEADERS = {"Content-Type": "application/json"} @@ -96,60 +97,46 @@ async def sync_alerting(context: dict, token: dict, userinfo: dict): ) -async def sync_datasource(context: dict, token: dict, userinfo: dict): - username = userinfo["preferred_username"] - prometheus_url = f"{DRYCC_CONTROLLER_API_URL}/v2/prometheus/{username}" - datasource_name = "Prometheus on Drycc" +async def sync_datasources(context: dict, token: dict, userinfo: dict): + headers = api_headers(context, userinfo) + datasources_path = os.path.join(os.path.dirname(__file__), "..", "datasources") + created, drycc_token = await _get_or_create_drycc_token(userinfo["preferred_username"], token) async with httpx.AsyncClient() as client: - resp = await client.get( - api_url("/api/datasources/name/{datasource_name}"), - headers=api_headers(context, userinfo)) - if resp.status_code == 200: - datasource = resp.json() - drycc_token_uuid = datasource.get("jsonData", {}).get("tokenId", None) - drycc_token = await _get_or_create_drycc_token(drycc_token_uuid, token) - if "token" in drycc_token: - datasource["jsonData"]["tokenId"] = drycc_token["uuid"] - datasource["secureJsonData"] = { - "httpHeaderValue1": f"Token {drycc_token["token"]}", - } - await _set_datasource_read_only(False, datasource["id"]) - await client.put( - api_url(f"/api/datasources/uid/{datasource["uid"]}"), - headers=api_headers(context, userinfo), - json=datasource, - ) - await _set_datasource_read_only(True, datasource["id"]) - return - drycc_token = await _get_or_create_drycc_token(None, token) - resp = await client.post( - api_url("/api/datasources"), - headers=api_headers(context, userinfo), - json={ - "name": datasource_name, - "type": "prometheus", - "url": prometheus_url, - "access": "proxy", - "basicAuth": False, - "jsonData": { - "tokenId": drycc_token["uuid"], - "httpHeaderName1": "Authorization", - "httpMethod": "GET", - "timeInterval": DRYCC_GRAFANA_REFRESH, - }, - "secureJsonData": { - "httpHeaderValue1": f"Token {drycc_token["token"]}", - }, - }, - ) - await _set_datasource_read_only(True, resp.json()["datasource"]["id"]) - - -async def sync_dashboard(context: dict, token: dict, userinfo: dict): - dashboard_path = os.path.join(os.path.dirname(__file__), "..", "dashboard") + for filename in os.listdir(datasources_path): + with open(os.path.join(datasources_path, filename)) as f: + template = Template(f.read()) + datasource = json.loads(template.substitute( + controller_api_url=DRYCC_CONTROLLER_API_URL, + username=userinfo["preferred_username"], + time_interval=DRYCC_GRAFANA_REFRESH, + token=drycc_token + )) + resp = await client.get( + api_url(f"/api/datasources/name/{datasource["name"]}"), headers=headers) + if resp.status_code == 200: + if created: + datasource = resp.json() + datasource["secureJsonData"] = { + "httpHeaderValue1": f"Token {drycc_token}", + } + await _set_datasource_read_only(False, datasource["id"]) + await client.put( + api_url(f"/api/datasources/uid/{datasource["uid"]}"), + headers=headers, json=datasource) + await _set_datasource_read_only(True, datasource["id"]) + elif resp.status_code == 404: + resp = await client.post( + api_url("/api/datasources"), headers=headers, json=datasource) + await _set_datasource_read_only(True, resp.json()["datasource"]["id"]) + else: + raise ValueError(f"grafana returned an unexpected status: {resp.status_code}") + + +async def sync_dashboards(context: dict, token: dict, userinfo: dict): + dashboards_path = os.path.join(os.path.dirname(__file__), "..", "dashboards") async with httpx.AsyncClient() as client: - for filename in os.listdir(dashboard_path): - with open(os.path.join(dashboard_path, filename)) as f: + for filename in os.listdir(dashboards_path): + with open(os.path.join(dashboards_path, filename)) as f: dashboard = json.load(f) dashboard.update({"id": None, "refresh": DRYCC_GRAFANA_REFRESH}) await client.post( @@ -172,21 +159,37 @@ async def _set_datasource_read_only(read_only: bool, id: str): await conn.commit() -async def _get_or_create_drycc_token(drycc_token_uuid: str, token: dict): - headers = {"Authorization": f"Bearer {token["access_token"]}"} - async with httpx.AsyncClient() as client: - if drycc_token_uuid: - resp = await client.get( - f"{DRYCC_CONTROLLER_API_URL}/v2/tokens/{drycc_token_uuid}", headers=headers) - if resp.status_code == 200: - return resp.json() - elif resp.status_code != 404: - raise ValueError( - f"the Drycc controller returned an unsupported status code {resp.status_code}" +async def _get_or_create_drycc_token(username, token: dict): + async def _check_or_create_drycc_token(drycc_token, token): + async with httpx.AsyncClient() as client: + created = False if drycc_token else True + if drycc_token: + headers = {"Authorization": f"Token {drycc_token}"} + resp = await client.get( + f"{DRYCC_CONTROLLER_API_URL}/v2/auth/whoami", headers=headers) + if resp.status_code in [401, 403]: + created = True + if created: + headers = {"Authorization": f"Bearer {token["access_token"]}"} + data = (await client.post( + f"{DRYCC_CONTROLLER_API_URL}/v2/auth/token/?alias=grafana-datasource", + headers=headers, json=token)).json() + drycc_token = data["token"] + return created, drycc_token + + async with await AsyncConnection.connect(os.environ.get("GF_DATABASE_URL")) as conn: + async with conn.cursor() as cursor: + await cursor.execute( + "SELECT o_auth_id_token FROM user_auth WHERE auth_module=%s AND auth_id=%s", + ("authproxy", username) + ) + drycc_token = (await cursor.fetchone())[0] + created, drycc_token = await _check_or_create_drycc_token(drycc_token, token) + if created: + async with conn.cursor() as cursor: + await cursor.execute( + "UPDATE user_auth SET o_auth_id_token=%s WHERE auth_module=%s AND auth_id=%s", + (drycc_token, "authproxy", username) ) - resp = await client.post( - f"{DRYCC_CONTROLLER_API_URL}/v2/auth/token/?alias=grafana-datasource", - headers=headers, - json=token, - ) - return resp.json() + await conn.commit() + return created, drycc_token From a81faf2829af42a24dbb0f9ed968482662cdf12e Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 15 Jul 2025 22:51:30 +0800 Subject: [PATCH 32/50] chore(grafana): simplify the connection controller --- charts/grafana/templates/_helpers.tmpl | 12 +++++++----- charts/grafana/values.yaml | 5 ----- .../grafana/oauth2/datasources/prometheus.json | 2 +- .../grafana/oauth2/datasources/quickwit.json | 2 +- .../hook/__pycache__/__init__.cpython-312.pyc | Bin 327 -> 0 bytes .../hook/__pycache__/grafana.cpython-312.pyc | Bin 10656 -> 0 bytes rootfs/usr/share/grafana/oauth2/hook/grafana.py | 13 +++++-------- 7 files changed, 14 insertions(+), 20 deletions(-) delete mode 100644 rootfs/usr/share/grafana/oauth2/hook/__pycache__/__init__.cpython-312.pyc delete mode 100644 rootfs/usr/share/grafana/oauth2/hook/__pycache__/grafana.cpython-312.pyc diff --git a/charts/grafana/templates/_helpers.tmpl b/charts/grafana/templates/_helpers.tmpl index 1c0599b..a5d169b 100644 --- a/charts/grafana/templates/_helpers.tmpl +++ b/charts/grafana/templates/_helpers.tmpl @@ -19,17 +19,19 @@ env: name: valkey-creds key: password - name: DRYCC_VALKEY_URL - value: "redis://:$(DRYCC_VALKEY_PASSWORD)@drycc-valkey.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:16379/0" + value: "redis://:$(DRYCC_VALKEY_PASSWORD)@drycc-valkey:16379/0" {{- end }} - name: DRYCC_SERVICE_KEY valueFrom: secretKeyRef: name: controller-creds key: service-key +- name: "DRYCC_CONTROLLER_URL" + value: http://drycc-controller-api - name: "DRYCC_QUICKWIT_URL" - value: "http://$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)/v2/quickwit/drycc" + value: "$(DRYCC_CONTROLLER_URL)/v2/quickwit/drycc" - name: "DRYCC_VICTORIAMETRICS_URL" - value: "http://$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)/v2/prometheus/drycc" + value: "$(DRYCC_CONTROLLER_URL)/v2/prometheus/drycc" {{- if .Values.passport.enabled}} - name: "DRYCC_PASSPORT_URL" {{- if .Values.global.certManagerEnabled }} @@ -72,7 +74,7 @@ env: name: database-creds key: password - name: GF_DATABASE_URL - value: "postgres://$(GF_DATABASE_USER):$(GF_DATABASE_PASSWORD)@drycc-database.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:5432/grafana" + value: "postgres://$(GF_DATABASE_USER):$(GF_DATABASE_PASSWORD)@drycc-database:5432/grafana" {{- end }} - name: "GF_SECURITY_ADMIN_USER" valueFrom: @@ -91,7 +93,7 @@ env: value: http://drycc-grafana.{{ .Values.global.platformDomain }} {{- end}} - name: GF_UNIFIED_ALERTING_HA_PEERS - value: "drycc-grafana.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:9094" + value: "drycc-grafana:9094" - name: GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS value: "$(POD_IP):9094" {{- range $key, $value := .Values.environment }} diff --git a/charts/grafana/values.yaml b/charts/grafana/values.yaml index ad89542..bb8e9d3 100644 --- a/charts/grafana/values.yaml +++ b/charts/grafana/values.yaml @@ -95,11 +95,6 @@ victoriametrics: enabled: true global: - # A domain name consists of one or more parts. - # Periods (.) are used to separate these parts. - # Each part must be 1 to 63 characters in length and can contain lowercase letters, digits, and hyphens (-). - # It must start and end with a lowercase letter or digit. - clusterDomain: "cluster.local" # The public resolvable hostname to build your cluster with. # # This will be the hostname that is used to build endpoints such as "drycc.$HOSTNAME" diff --git a/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json b/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json index 450d976..63c50e0 100644 --- a/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json +++ b/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json @@ -1,7 +1,7 @@ { "name": "Prometheus on Drycc", "type": "prometheus", - "url": "${controller_api_url}/v2/prometheus/${username}", + "url": "${controller_url}/v2/prometheus/${username}", "access": "proxy", "isDefault": true, "basicAuth": false, diff --git a/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json b/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json index 99e7fff..389e9f2 100644 --- a/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json +++ b/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json @@ -1,7 +1,7 @@ { "name": "Application Logs", "type": "quickwit-quickwit-datasource", - "url": "${controller_api_url}/v2/quickwit/${username}", + "url": "${controller_url}/v2/quickwit/${username}", "access": "proxy", "basicAuth": false, "jsonData": { diff --git a/rootfs/usr/share/grafana/oauth2/hook/__pycache__/__init__.cpython-312.pyc b/rootfs/usr/share/grafana/oauth2/hook/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 6609f3de5f3920434aa49e7daf97795f7ffd481c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 327 zcmXv}Jxjzu5Zz5&Zxe~fS@;V?h@klcVsCTJAwrt4-j17K+zq=)k;aeCPCNYzhX|Jb z2OCRSECee%;W{ffAx<&#=FOYo&1)2e$i?yFEWdU2t84y)3}CtKz&+xK;}Si27y%YI z^}vS}Ag~G}K~gGE8nj;Qe`6<(vBf*nIJ6_7*(sCE(n1z>rnIo6L&gTCmGi7Ipa-Z` z33lAGoYl;zM$f@UJ;6%Qbs>daALb@MQA~47a;46Ur5sGHRcjV#*_r;`O^)1UlkkSg zTuIRu$*ez^u&PSBr6kLGOW9u4TDpI(cvHfGp1AGq1GDAQkB>3_K%1YzP94mkf&pthw123izL%UALCRKP${p!XkB9bm<{ z0)21zX<4?D7X70u&Fq^uv$Hd^^WOWtH~h!EJOcyQfBjwWiQiQ)%)enFeF-UI$*y6T zD~!m9WSHqs-bp{9Wlg_^-r0Uu)QD`5^m74bi1wN8CV3yqd%TP#krTCFLf?4x3nD+1 zpQLm1AJCidB}qQ{1D?^PpUI3v4^J27jvz@7l|0FNO3Tvk-Lia@mgdZeQnL~DnXy?! zL%&`u5sg13{Z`Qw$UmkL^M1!{%;uX9? z;FYZ46#>tV{lj~Ufi4{<&eA{PB)T4%JfV?rU?kk!H$ED8Wc81Z4hIMQ;b3UQH6({d z)FN-s5sznJZ;#vO?s593mp;jkbdb+#%db$rB z==QKIk-P0FgTUuO^s#M3KG(I@k($?D2*0LG0ZJSy~ zf}<_L&|qr!(eHpX3u&ngvMWrKNLrCZnFy468Bvp_oNQMlnCkb)%k9!ZPq*jLfv&y1 z;kH9ZezdpuaBq0anb4Un$A`8Y*?v+!c6OWW-SpbaM_T=-&h~YV_`46g{o%1*fA45p zkM9LKvgYHG|CoQozsd+B4Y^ZzMnfm3^2b8bDSueigk)6+jGPWiufK-j5Kbz{=- zz^FfbLe&N30l#=EI3nYO9gauqU5Dg=B)h~h|Hz5Z$no)@t2ZA>Ksu8)Jc+Zx#sWP9c7qBeqqReSV z$`7Z62c7^Pwn&Q=^rWXxYI+%`p-aM-O89owHYx><1tcjT4vfKqjQCFlBF3J;sZco3 zj4PBWbU}@(4V>}^ha;v`gH9}|d`LPz5ELI-Uu)i*{G`p@((&f~qLZks9G0A{RDiG2 z;R}Kz$3m)bBH$Nc^<^A~!{LyuSZ~BdPSr>iz^R2KzXjQOCT^)zs@fFuhVy&lm1~vy z?Xk)o=l4#&9OKHKD6grXIul#tg6ggqR~{EEm)c`O*Y3XlZMPFh_&bP^~@ldW9L1VnWOx~DH%)4MJ&cXD4rMbfX zOsi!k*eE-sO;zXI!fAuUh-{Ski1`G*`HhT$h-z-|X+`)h7$Mpto9dB^azpx@u2N)r z3=`U@Rut$yAtxK*d)xGA#^5j0Bbc$=F%o8+fl(Tt?DKh}6on`Wk{h}&@q@y(a;)Tg zyt$pZ_#vcIkbIEJA?4LFZOpqkO4zHvqVsNNw2Vy7Xb5uzz6*}{EPI4GLw?PkAx?eH z&W;#Bl7lYTo3g9#3nE!ySa3t@qjI>-ztE%#sHvU2szsf1RyDfi@sUB#a1hkGg!Y@t zyi|kFv7=*QCoiE@lpL7R-uOYF zIl)4CIbndRM%9KxCj%oA8nwaHo&=I%bdacI&txR~|4^byTK|SKavBA#o|eBG z8EU1xOrUJ9B8QL3cG5-wpiF9ZQH!!J?E(p21$O30Fb~#&!?qtWqVNeT>TYDf*6QOF z4PZZL?;zvG|D}w70&@K$NTraBKTy6+T@gK%YP2L$6-+_Z1VvTAB3K~R1Xf-i8x2Uv zh~(3z)atvCA-f_b7gU}K$}%7{xldYyJ(z=X%51q2AV}4~bP7@}+@w0pvPuAr64eCK zT5QBqyC$teF77l5L4ni&naa@@u2chhR4vrbN^Q_;2PFBQAftNV8NzRzcWnCmlXn}G z-q+_l-ayfRbHVZ^h`y!ZgYoyre|`?g%Ic}QYyFeLC4Ed+7B?1L>bhsF1?L&8>Y01T3sPLU2zd+#1rM^>?pM9QAjunG{P&c z9L1FP{5L~yW*nzz5v?ChyugEkUIvyP9mXhe5lS0FRq|(!5k&RRT zYz3(DVqiEB4yfE{2q8Yaz^QwNI>h0qc_;iSFN{JVz@&_f2AKk-3|2}iNjj=sP`M9V z1z<6rqa>Bat@Dm;b35*yQC@p{-s49td33>Y^l>d|ubj;LLqW-;CvLG_YJWd^^+)$C zj<}=la^8cwhH2r`1DAxWx_j2@gq|t1e`{b&rBjX59rtRs-#l?Ea<8KY6>9V4W~xxv zPamE+5^LVAtbakVyYG7MTVKg)(}b2Oa4az_Do{4h{4LFNjIC;F-}GDe>UQ4J-7UNK zLLc_mbh#<1pl8DOtvj<_#OqrWmnT;5y?Z=X^eSrPH7hl8RT7>3sgW@jTy24Q=-d-1K?`A06X(!$`{?1l63x(Mo#JiE7-N^wx#~9&(IV16I=H^Ug zNY|526F1kuBkdyCrj6zsX}*~tzn$;Y3v;_jCu5rH5P<$#s{{UH${l?tiriiWBJGo9 z)jJ5W28B3{xWJD_k1H4ZK`mZk^n2-*Zi`|~MUw53>`+6+0K8W0Pvyr(ML%qGi?`F~ zK6IdOU|*-(+u76WeP4movWC^^>j-cmIy7{f>5cp~3n1VxS(Si*Xbtm|P6dUIYg`!G9)^wUim0LlyOwEPh_ow$fXxEoPn*6LEVf?o6)Ih zHx}&MkSW_|ni6?x?Y@B8Rjft^2u>Xc6Vhrx|<;1gUJ!s9Rm34(?G!NGtmtNNpUIXH+fk-X4ELNo6)srFUF;D>q+ zJQa?pMgT?OF?nDx1R+HlrYReM@Gn4&fk7z%s(XNjn+K8+0d(A??UGHOS(52-6@CM?uUWIK^C=ox?{KS~3i zxVcy^sST{{_37#=o7mVdGWBFD2p0OrgT6w+nTIomSlP|`roY#MR?Z?wY zH*4mrb}dx7V^!{3&bxi{RXvKSV$rc(+40idL-UT_g|gnsm*SS+W4m3xA@qm zy_XDEx5upYQ{ypf+k?XT>H3-Sn}=g{H zURm>?YTfjfSk;Dj!Fm?vn zLj+#Q*w}Xo4J`C`2@_M^N0t*N81HhT91&Qeo+c~&Wq+3c0rtR!CZCJ^ zn)r15-?Bh`wa!<@-YzFTBY(TXT?mCct;A>M@3gtCP?#gcXXWQKTBHRdJTPY?J{vbz zphenlL4E`A74mb9ZUYp)-avfC{MQ>fq}%aTjl- z)7(xuWxQ{_@Qs7`YE9o%3rIH#(DE@5hi9Y|xBMdOp#v*&H}a5 ziYv-SGbk$E?`CF#4`;B&YG=4nKHU~039FsxicA<$FTF(e;|%^F3ce0hy>d|>#3-~P4}Q9kUh53{m3+#9`+VR}ufQ!G$c-cSH z#_PHwnwAz-FZ$&ZM?-!|bZVsaFeO!h6EX0s!JzrCG0|Uo`!-0W;tWi^i;h-Fb1}YaCg}^nnW884a zFG{^mObAe_J(YhDHxPsew{;_$q49$T2$9Ra|a!%KoG#E}epRz;l{YTG@mbC14WBlA?)udAo5uN?9XI=K?U*m>KHv3(0a~<|T{m1a zOm+N=eZ%Cw#r&cVk6t-``S_LL%fnM0Gh642cizw6bt`|d!f}27wf$56j}BbY$E~Fc zR!7Y0nC9+p>xxx(-?tu&7gt<4dHE#Tg^Jes!nUj_c(Ar5zSjAzmMtqrlTfq72xt-n zGzpMB6c}5FlD`WMElM{&#FLApHU<|MYM0a1kH8-0`oe#fQ8q8Q%R9B?b|vwy;cu^D zk*;=YysZ$rbeDT;gxL<_br@!Mb3o6r2B?_JGeWC5JMp$~bEQ1e4uZ8cT5pqau0g;y zEd=?ke5Xd3+d{nCOmpo5(%|-L$?AuY_)CM%IzVN7UpG!V2~c|AjS!EvK> z4qW6_G)))-*YpAplB36xEF48AUgabjj&_j z?Ax{5KOFSSyHdZH$nk`d%21b8bjj0{Lof&l135`WXP`b!rzC|Ws^pwn=?$!;Co5=y zvRGTj?MlHjh0fApti|jzNODmIE?d@2HB7IaF~`>JR-7*=4!2_7bAIm=I_mev@_1?0 z^|EVaiv>-JZT<91%GP~K`+mjxqSE&|D2}qaC5Du}Nggt!@J%x5ePAhAuvEn?RZ|=9 zS?c2@-_;&2z+e)`zDLlHw zYDz#!X+Tr)4CuGlFTcKMX;93KO5;Jr^a>uz=A%w4MwnZS2eMWJ;B*6HEli&HJWPfD z?!m!qg2S;bC)HS@5SlJ0idLw!_h5N0)Oro%uZX9K|7#YgFKRrS+1p0qY36U6DBWTL z_U-M&1ChQRTA=SRZY?yL)e=t|Kg$;&U5brn*Afo|{pxw7n+ftExCe9_r6ITn^k&{G z2oN6hbeLv$3P8`X0`Q&02Uc?v3`o^d*T`8$jiuxRR`PhQ7O597!^lu_&)|ob5I{Vitf5exY+zMq4W#!&1rPzq|6fd-v_>cK3McS-#4N!ND-Kp)_z!&*CJEHPZkc zb-t(xr2Fa%_~8Ur(VwX3cAg4}W5a_1D%f+v@F6Y-P88i5pCEd(1$3YH3u#E_^k5cAc(1OvC}I(oYayB$i2vLez$ zunO;3hj*;Rd!kiC>ZS@|_maA4k=|~K;7*ddgrSKTrZy!QxXp0%c8goWQsNC_CblW) b32xJ2db=53!cw9ppOoJe|HQzJF7 Date: Wed, 16 Jul 2025 22:40:54 +0800 Subject: [PATCH 33/50] chore(grafana): change prometheus http method to POST --- rootfs/usr/share/grafana/oauth2/datasources/prometheus.json | 2 +- rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json b/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json index 63c50e0..54bbdbe 100644 --- a/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json +++ b/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json @@ -7,7 +7,7 @@ "basicAuth": false, "jsonData": { "httpHeaderName1": "Authorization", - "httpMethod": "GET", + "httpMethod": "POST", "timeInterval": "${time_interval}" }, "secureJsonData": { diff --git a/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml b/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml index c64470a..9c9981f 100644 --- a/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml +++ b/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml @@ -21,6 +21,7 @@ datasources: url: $DRYCC_VICTORIAMETRICS_URL isDefault: true jsonData: + httpMethod: POST httpHeaderName1: X-Drycc-Service-Key secureJsonData: httpHeaderValue1: $DRYCC_SERVICE_KEY From a954edf2e90ac7a689acf25eba7f123e423ea879 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Thu, 17 Jul 2025 13:12:54 +0800 Subject: [PATCH 34/50] feat(grafana): add drycc application logs --- .../dashboards/drycc_application_logs.json | 318 ++ .../share/grafana/oauth2/dashboards/node.json | 3613 ----------------- .../oauth2/datasources/prometheus.json | 1 + .../grafana/oauth2/datasources/quickwit.json | 1 + .../provisioning/datasources/drycc.yaml | 2 + 5 files changed, 322 insertions(+), 3613 deletions(-) create mode 100644 rootfs/usr/share/grafana/oauth2/dashboards/drycc_application_logs.json delete mode 100644 rootfs/usr/share/grafana/oauth2/dashboards/node.json diff --git a/rootfs/usr/share/grafana/oauth2/dashboards/drycc_application_logs.json b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_application_logs.json new file mode 100644 index 0000000..1d4d8ea --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_application_logs.json @@ -0,0 +1,318 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Application Logs. See also the Explore tab for ad-hoc log queries.", + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 61, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "drycc" + ], + "targetBlank": false, + "title": "Drycc", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "datasource": { + "type": "quickwit-quickwit-datasource", + "uid": "application_logs" + }, + "description": "[Query Language](https://quickwit.io/docs/reference/query-language)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "interval": "1m", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "quickwit-quickwit-datasource", + "uid": "application_logs" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "kubernetes.namespace_name:${app:singlequote} kubernetes.pod_name:${instance:singlequote} ${query:text}", + "refId": "A", + "timeField": "" + } + ], + "title": "Logs volume", + "type": "timeseries" + }, + { + "datasource": { + "type": "quickwit-quickwit-datasource", + "uid": "application_logs" + }, + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": true, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "quickwit-quickwit-datasource", + "uid": "application_logs" + }, + "metrics": [ + { + "id": "1", + "settings": {}, + "type": "logs" + } + ], + "query": "kubernetes.namespace_name:${app:singlequote} kubernetes.pod_name:${instance:singlequote} ${query:text}", + "refId": "A", + "timeField": "" + } + ], + "title": "Application Logs", + "type": "logs" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "", + "value": "" + }, + "label": "Query", + "name": "query", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "type": "textbox" + }, + { + "allValue": "*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "definition": "label_values(namespace)", + "includeAll": true, + "label": "App", + "name": "app", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "definition": "label_values({namespace=\"$app\"},pod)", + "includeAll": true, + "label": "Instance", + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({namespace=\"$app\"},pod)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 5, + "type": "query" + }, + { + "allowCustomValue": true, + "baseFilters": [], + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "description": "", + "filters": [], + "hide": 2, + "name": "filter", + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Drycc Logs", + "uid": "aepz8ewtzq4g0f", + "version": 20 +} \ No newline at end of file diff --git a/rootfs/usr/share/grafana/oauth2/dashboards/node.json b/rootfs/usr/share/grafana/oauth2/dashboards/node.json deleted file mode 100644 index f76826d..0000000 --- a/rootfs/usr/share/grafana/oauth2/dashboards/node.json +++ /dev/null @@ -1,3613 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "cpu memroy disk IO network temperature", - "editable": false, - "fiscalYearStartMonth": 0, - "gnetId": 8919, - "graphTooltip": 0, - "id": null, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [], - "targetBlank": true, - "title": "More", - "tooltip": "", - "type": "dashboards", - "url": "" - } - ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "prometheus" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 176, - "panels": [], - "title": "Basic info", - "type": "row" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "editable": false, - "error": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 11, - "links": [], - "maxPerRow": 6, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "", - "mode": "html" - }, - "pluginVersion": "9.5.3", - "repeat": "node", - "repeatDirection": "h", - "style": {}, - "title": "$node", - "type": "text" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 1 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 2 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 2, - "x": 0, - "y": 2 - }, - "hideTimeOverride": true, - "id": 15, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "time() - node_boot_time_seconds{instance=~\"$node\"}", - "format": "time_series", - "hide": false, - "instant": true, - "intervalFactor": 2, - "refId": "A", - "step": 40 - } - ], - "title": "uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 1 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 2 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 2, - "x": 2, - "y": 2 - }, - "id": 14, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "expr": "count(count(node_cpu_seconds_total{instance=~\"$node\", mode='system'}) by (cpu))", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 20 - } - ], - "title": "cpu cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 2 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 3 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 2, - "x": 4, - "y": 2 - }, - "id": 75, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A", - "step": 20 - } - ], - "title": "memory", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 2, - "x": 6, - "y": 2 - }, - "hideTimeOverride": false, - "id": 171, - "links": [], - "maxDataPoints": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "value" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "exemplar": true, - "expr": "sum(node_filesystem_size_bytes {instance=~\"$node\",fstype=~\"ext4|xfs\"})", - "format": "table", - "hide": false, - "instant": false, - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{mountpoint}}", - "refId": "A" - } - ], - "title": "system disk", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 50 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 8, - "y": 2 - }, - "id": 167, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "expr": "100 - (avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"idle\"}[5m])) * 100)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 20 - } - ], - "title": "cpu usage（5m）", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 20 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 50 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 11, - "y": 2 - }, - "id": 20, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"iowait\"}[5m])) * 100", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 20 - } - ], - "title": "CPU iowait（5m）", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 80 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 14, - "y": 2 - }, - "hideTimeOverride": false, - "id": 172, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "expr": "(1 - (node_memory_MemAvailable_bytes{instance=~\"$node\"} / (node_memory_MemTotal_bytes{instance=~\"$node\"})))* 100", - "format": "time_series", - "hide": false, - "interval": "10s", - "intervalFactor": 1, - "refId": "A", - "step": 20 - } - ], - "title": "Memory Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "Obtain the maximum partition through the variable maxmount.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 70 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 17, - "y": 2 - }, - "id": 154, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "9.5.3", - "repeatDirection": "h", - "targets": [ - { - "expr": "100 - ((node_filesystem_avail_bytes{instance=~\"$node\",mountpoint=\"$maxmount\",fstype=~\"ext4|xfs\"} * 100) / node_filesystem_size_bytes {instance=~\"$node\",mountpoint=\"$maxmount\",fstype=~\"ext4|xfs\"})", - "format": "time_series", - "interval": "10s", - "intervalFactor": 1, - "refId": "A", - "step": 20 - } - ], - "title": "partion ($maxmount) usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [], - "max": 10000, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 7000 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 9000 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 21, - "y": 2 - }, - "hideTimeOverride": false, - "id": 16, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "expr": "node_filefd_allocated{instance=~\"$node\"}", - "format": "time_series", - "instant": false, - "interval": "10s", - "intervalFactor": 1, - "refId": "B" - } - ], - "title": "opend file descriptor", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "15 minutes" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "1 minute" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "5分钟" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CCA300", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 11, - "x": 0, - "y": 7 - }, - "id": 13, - "links": [], - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.2.3", - "targets": [ - { - "expr": "node_load1{instance=~\"$node\"}", - "format": "time_series", - "instant": false, - "interval": "10s", - "intervalFactor": 2, - "legendFormat": "{{instance}}_1m", - "metric": "", - "refId": "A", - "step": 20, - "target": "" - }, - { - "expr": "node_load5{instance=~\"$node\"}", - "format": "time_series", - "instant": false, - "interval": "10s", - "intervalFactor": 2, - "legendFormat": "{{instance}}_5m", - "refId": "B", - "step": 20 - }, - { - "expr": "node_load15{instance=~\"$node\"}", - "format": "time_series", - "instant": false, - "interval": "10s", - "intervalFactor": 2, - "legendFormat": "{{instance}}_15m", - "refId": "C", - "step": 20 - } - ], - "title": "Load average", - "type": "timeseries" - }, - { - "aliasColors": { - "Idle - Waiting for something to happen": "#052B51", - "guest": "#9AC48A", - "idle": "#052B51", - "iowait": "#EAB839", - "irq": "#BF1B00", - "nice": "#C15C17", - "sdb_I/O operations per second%": "#d683ce", - "softirq": "#E24D42", - "steal": "#FCE2DE", - "system": "#508642", - "user": "#5195CE", - "The proportion of disk spent on I/O operations": "#ba43a9" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "decimals": 2, - "description": "node_disk_io_time_seconds_total", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 13, - "x": 11, - "y": 7 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxPerRow": 6, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"system\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}_System", - "refId": "A", - "step": 20 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"user\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}_User", - "range": true, - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"idle\"}[2m])) by (instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{instance}}_Idle", - "range": true, - "refId": "F", - "step": 240 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"iowait\"}[2m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}_Iowait", - "range": true, - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}_{{device}}_per second I/O%", - "range": true, - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU usage、disk I/O time（%）", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:513", - "format": "percentunit", - "label": "", - "logBase": 1, - "max": "1", - "show": true - }, - { - "$$hashKey": "object:514", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "displayName": "", - "unit": "bytes" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 11, - "x": 0, - "y": 14 - }, - "hiddenSeries": false, - "id": 207, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"}-node_filesystem_avail_bytes {instance=~'$node',fstype=~\"ext4|xfs\"}", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{mountpoint}}_usage", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Partion usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transformations": [], - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3570", - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:3571", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": false, - "inspect": false - }, - "decimals": 2, - "displayName": "", - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "mountpoint" - }, - "properties": [ - { - "id": "displayName", - "value": "partion" - }, - { - "id": "unit", - "value": "bytes" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)" - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #A" - }, - "properties": [ - { - "id": "displayName", - "value": "free" - }, - { - "id": "unit", - "value": "bytes" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 10000000000 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 20000000000 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #B" - }, - "properties": [ - { - "id": "displayName", - "value": "usage" - }, - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "color-background" - } - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 70 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #C" - }, - "properties": [ - { - "id": "displayName", - "value": "total" - }, - { - "id": "unit", - "value": "bytes" - }, - { - "id": "decimals", - "value": 1 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "fstype" - }, - "properties": [ - { - "id": "displayName", - "value": "file system" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "displayName", - "value": "file system" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "device" - }, - "properties": [ - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 13, - "x": 11, - "y": 14 - }, - "id": 164, - "links": [], - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "file system" - } - ] - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "exemplar": false, - "expr": "node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "C" - }, - { - "exemplar": true, - "expr": "node_filesystem_avail_bytes {instance=~'$node',fstype=~\"ext4|xfs\"}", - "format": "table", - "hide": true, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - }, - { - "exemplar": false, - "expr": "1-(node_filesystem_free_bytes{instance=~'$node',fstype=~\"ext4|xfs\"} / node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "B" - } - ], - "title": "Disk usage", - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "fstype", - "mountpoint", - "Value #C", - "Value #A", - "Value #B", - "device" - ] - } - } - } - ], - "type": "table" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 211, - "panels": [ - { - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "MEM_Avaliable" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "MEM_Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "MEM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "MEM_Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6d1f62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "MEM_Used" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#eab839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "avaliable" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9ac48a", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "totol memery" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#bf1b00", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 2 - }, - "id": 156, - "links": [], - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"}", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}_TotalMem", - "refId": "A", - "step": 4 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"} - node_memory_MemAvailable_bytes{instance=~\"$node\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}_Used", - "range": true, - "refId": "B", - "step": 4 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_MemFree_bytes{instance=~\"$node\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "MEM_Free", - "range": true, - "refId": "C", - "step": 4 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_Buffers_bytes{instance=~\"$node\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "MEM_Buffers", - "range": true, - "refId": "D", - "step": 4 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_Cached_bytes{instance=~\"$node\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "MEM_Cached", - "range": true, - "refId": "E", - "step": 4 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_MemAvailable_bytes{instance=~\"$node\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}_Available", - "range": true, - "refId": "F", - "step": 4 - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"} - (node_memory_Cached_bytes{instance=~\"$node\"} + node_memory_Buffers_bytes{instance=~\"$node\"} + node_memory_MemFree_bytes{instance=~\"$node\"})", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "refId": "G" - } - ], - "title": "Memory info", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 2 - }, - "id": 206, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.2.3", - "targets": [ - { - "exemplar": true, - "expr": "node_memory_SwapFree_bytes{instance=~\"$node\"}", - "hide": false, - "interval": "", - "legendFormat": "Swap_Free", - "refId": "A" - }, - { - "exemplar": true, - "expr": "node_memory_SwapCached_bytes{instance=~\"$node\"}", - "hide": false, - "interval": "", - "legendFormat": "SwapCached", - "refId": "B" - }, - { - "exemplar": true, - "expr": "node_memory_SwapTotal_bytes{instance=~\"$node\"}", - "hide": false, - "interval": "", - "legendFormat": "SwapTotal", - "refId": "C" - } - ], - "title": "Swap memory", - "type": "timeseries" - } - ], - "title": "Memory", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 209, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 3 - }, - "height": "300", - "hiddenSeries": false, - "id": 157, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:101", - "alias": "/.*_out transmit$/", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_network_receive_bytes_total{instance=~'$node',device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*|lxc.*|nodelocaldns|cilium.*'}[5m])*8", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}_{{device}}_in receive", - "range": true, - "refId": "A", - "step": 4 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_network_transmit_bytes_total{instance=~'$node',device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*|lxc.*|nodelocaldns|cilium.*'}[5m])*8", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}_{{device}}_out transmit", - "range": true, - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Network flow", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:85", - "format": "bps", - "label": "upload（-）/download（+）", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:86", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "TCP": "#6ED0E0" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 3 - }, - "height": "300", - "hiddenSeries": false, - "id": 158, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node_netstat_Tcp_CurrEstab{instance=~'$node'}", - "format": "time_series", - "hide": false, - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{instance}}_ESTABLISHED", - "refId": "A", - "step": 20 - }, - { - "expr": "node_sockstat_TCP_tw{instance=~'$node'}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}_TCP_tw", - "refId": "D" - }, - { - "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=~'$node'}[1m])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}_ActiveOpens", - "refId": "B" - }, - { - "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=~'$node'}[1m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}_PassiveOpens", - "refId": "C" - }, - { - "expr": "node_sockstat_TCP_alloc{instance=~'$node'}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}_TCP_alloc", - "refId": "E" - }, - { - "expr": "node_sockstat_TCP_inuse{instance=~'$node'}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}_TCP_inuse", - "refId": "F" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TCP connect info", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:269", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:270", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "title": "Network", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 190, - "panels": [ - { - "aliasColors": { - "vda_write": "#6ED0E0" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "Reads completed: Reads completed per second for each disk partition\n\nWrites completed: \nIO now", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 24 - }, - "height": "300", - "hiddenSeries": false, - "id": 161, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:386", - "alias": "/.*_read$/", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_reads_completed_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}_{{device}}_read", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_writes_completed_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}_{{device}}_write", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "node_disk_io_now{instance=~\"$node\"}", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{device}}", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "disk read and write（IOPS）", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:393", - "format": "iops", - "label": "read（-）/write（+）I/O ops/sec", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:394", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "vda": "#6ED0E0" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "Read time ms The number of seconds taken for each disk partition read operation\n\nWrite time ms\nIO time ms \nIO time weighted", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 3, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 24 - }, - "height": "300", - "hiddenSeries": false, - "id": 160, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:459", - "alias": "/,*_read$/", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{device}}", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{device}}_weighting", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_read_time_seconds_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}_{{device}}_read", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_write_time_seconds_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}_{{device}}_write", - "range": true, - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "disk io read write time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:466", - "format": "s", - "label": "read（-）/write（+）", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:467", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "The number of read operations per second=[Δrd_ios/Δt]", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 31 - }, - "hiddenSeries": false, - "id": 194, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "rate(node_disk_reads_completed_total{instance=~'$node'}[2m])", - "interval": "", - "legendFormat": "{{device}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "r/s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1217", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1218", - "format": "bytes", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "The number of write operations per second=[Δwr_ios/Δt]", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 31 - }, - "hiddenSeries": false, - "id": 196, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "(rate(node_disk_writes_completed_total{instance=~\"$node\"}[2m]))", - "interval": "", - "legendFormat": "{{device}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "w/s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1363", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1364", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "The average time required for each read operation=[ Δ Rd_ticks/ Δ Rd_ios not only includes the time required for hard disk device read operations, but also the waiting time in the kernel queue.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 39 - }, - "hiddenSeries": false, - "id": 202, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "(rate(node_disk_read_time_seconds_total {instance=~'$node'}[2m])) /(rate(node_disk_reads_completed_total {instance=~'$node'}[2m]))", - "interval": "", - "legendFormat": "{{device}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "r_await", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:2148", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:2149", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "Average time required for each write operation=[ Δ Wr_ticks/ Δ Wr_ios not only includes the time required for hard disk device write operations, but also the waiting time in the kernel queue.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 39 - }, - "hiddenSeries": false, - "id": 204, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "(rate(node_disk_write_time_seconds_total {instance=~'$node'}[2m])) /(rate(node_disk_writes_completed_total {instance=~'$node'}[2m]))", - "interval": "", - "legendFormat": "{{device}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "w_await", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:2477", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:2478", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 47 - }, - "hiddenSeries": false, - "id": 198, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "(rate(node_disk_read_bytes_total {instance=~'$node'}[2m]))", - "interval": "", - "legendFormat": "{{device}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "rMiB/s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1622", - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1623", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 47 - }, - "hiddenSeries": false, - "id": 200, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "(rate(node_disk_written_bytes_total {instance=~'$node'}[2m]))", - "interval": "", - "legendFormat": "{{device}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "wKiB/s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1920", - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1921", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "vda_write": "#6ED0E0" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 55 - }, - "height": "300", - "hiddenSeries": false, - "id": 168, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*_read$/", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_read_bytes_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}_{{device}}_read", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_written_bytes_total{instance=~\"$node\"}[2m])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}_{{device}}_write", - "range": true, - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Disk read and write capacity size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": "read（-）/write（+）", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "description": "The busy ratio of this hard drive device=[ Δ Io_ticks/ Δ t] The time ratio indicating that the device has I/O (i.e. non idle), regardless of how much I/O there is, only considering whether there is", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 65 - }, - "hiddenSeries": false, - "id": 192, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeat": "idc", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "rate(node_disk_io_time_seconds_total{instance=~\"$node\"}[2m])", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{device}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "%util", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:744", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:745", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "title": "Disk read write", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 213, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus" - }, - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 25 - }, - "hiddenSeries": false, - "id": 169, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.5.3", - "pointradius": 0.5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "editorMode": "code", - "expr": "node_hwmon_temp_celsius{instance=~'$node'}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{chip}}_{{sensor}}", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "cpu temperature", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3976", - "format": "celsius", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:3977", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "title": "Cpu temperature", - "type": "row" - } - ], - "refresh": "", - "schemaVersion": 38, - "style": "dark", - "tags": ["instance"], - "templating": { - "list": [ - { - "auto": true, - "auto_count": 10, - "auto_min": "10s", - "current": { - "selected": true, - "text": "5s", - "value": "5s" - }, - "hide": 0, - "label": "interval", - "name": "interval", - "options": [ - { - "selected": false, - "text": "auto", - "value": "$__auto_interval_interval" - }, - { - "selected": true, - "text": "5s", - "value": "5s" - }, - { - "selected": false, - "text": "10s", - "value": "10s" - }, - { - "selected": false, - "text": "30s", - "value": "30s" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - } - ], - "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - }, - { - "allFormat": "glob", - "allValue": "", - "current": { - "selected": false, - "text": [], - "value": [] - }, - "datasource": { - "type": "prometheus" - }, - "definition": "label_values(node_exporter_build_info{},instance)", - "hide": 0, - "includeAll": false, - "label": "host instance", - "multi": true, - "multiFormat": "regex values", - "name": "node", - "options": [], - "query": { - "query": "label_values(node_exporter_build_info{},instance)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "/", - "value": "/" - }, - "datasource": { - "type": "prometheus" - }, - "definition": "", - "hide": 2, - "includeAll": false, - "label": "", - "multi": false, - "name": "maxmount", - "options": [], - "query": { - "query": "query_result(topk(1,sort_desc (max(node_filesystem_size_bytes{instance=~'$node',fstype=~\"ext4|xfs\"}) by (mountpoint))))", - "refId": "Prometheus-maxmount-Variable-Query" - }, - "refresh": 2, - "regex": "/.*\\\"(.*)\\\".*/", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "datasource": {}, - "filters": [], - "hide": 0, - "label": "", - "name": "Filters", - "skipUrlSync": false, - "type": "adhoc" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] - }, - "timezone": "browser", - "title": "Node Metrics", - "uid": "SQlD2vTZq", - "version": 1, - "weekStart": "" -} diff --git a/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json b/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json index 54bbdbe..4eefc33 100644 --- a/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json +++ b/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json @@ -1,6 +1,7 @@ { "name": "Prometheus on Drycc", "type": "prometheus", + "uid": "prometheus_on_drycc", "url": "${controller_url}/v2/prometheus/${username}", "access": "proxy", "isDefault": true, diff --git a/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json b/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json index 389e9f2..8c1861b 100644 --- a/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json +++ b/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json @@ -1,6 +1,7 @@ { "name": "Application Logs", "type": "quickwit-quickwit-datasource", + "uid": "application_logs", "url": "${controller_url}/v2/quickwit/${username}", "access": "proxy", "basicAuth": false, diff --git a/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml b/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml index 9c9981f..d9b9a66 100644 --- a/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml +++ b/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml @@ -4,6 +4,7 @@ datasources: type: quickwit-quickwit-datasource access: proxy orgId: 1 + uid: application_logs url: $DRYCC_QUICKWIT_URL jsonData: httpHeaderName1: X-Drycc-Service-Key @@ -18,6 +19,7 @@ datasources: type: prometheus access: proxy orgId: 1 + uid: prometheus_on_drycc url: $DRYCC_VICTORIAMETRICS_URL isDefault: true jsonData: From 8f0d20108acad24c14220756cd83277e6240bb26 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 18 Jul 2025 10:58:23 +0800 Subject: [PATCH 35/50] chore(grafana): add drycc folder --- rootfs/usr/share/grafana/oauth2/hook/grafana.py | 6 ++++++ rootfs/usr/share/grafana/provisioning/dashboards/drycc.yaml | 2 ++ 2 files changed, 8 insertions(+) diff --git a/rootfs/usr/share/grafana/oauth2/hook/grafana.py b/rootfs/usr/share/grafana/oauth2/hook/grafana.py index ac7dcf1..d13647e 100644 --- a/rootfs/usr/share/grafana/oauth2/hook/grafana.py +++ b/rootfs/usr/share/grafana/oauth2/hook/grafana.py @@ -132,6 +132,11 @@ async def sync_datasources(context: dict, token: dict, userinfo: dict): async def sync_dashboards(context: dict, token: dict, userinfo: dict): dashboards_path = os.path.join(os.path.dirname(__file__), "..", "dashboards") async with httpx.AsyncClient() as client: + await client.post( + api_url("/api/folders"), + headers=api_headers(context, userinfo), + json={"uid": "drycc", "title": "drycc"}, + ) for filename in os.listdir(dashboards_path): with open(os.path.join(dashboards_path, filename)) as f: dashboard = json.load(f) @@ -141,6 +146,7 @@ async def sync_dashboards(context: dict, token: dict, userinfo: dict): headers=api_headers(context, userinfo), json={ "dashboard": dashboard, + "folderUid": "drycc", "overwrite": True, }, ) diff --git a/rootfs/usr/share/grafana/provisioning/dashboards/drycc.yaml b/rootfs/usr/share/grafana/provisioning/dashboards/drycc.yaml index b9ca478..a047e87 100644 --- a/rootfs/usr/share/grafana/provisioning/dashboards/drycc.yaml +++ b/rootfs/usr/share/grafana/provisioning/dashboards/drycc.yaml @@ -2,6 +2,8 @@ apiVersion: 1 providers: - name: 'default' orgId: 1 + folder: 'drycc' + folderUid: 'drycc' type: file options: path: /usr/share/grafana/dashboards From 1c9125f2869a6c3ec59a8dace5ed683bc4ade571 Mon Sep 17 00:00:00 2001 From: lijianguo Date: Fri, 18 Jul 2025 11:59:39 +0800 Subject: [PATCH 36/50] chore(grafana): add alert rule (#39) --- .../grafana/oauth2/alerting/pod_cpu.json | 80 ++ .../grafana/oauth2/alerting/pod_memory.json | 81 ++ .../grafana/oauth2/alerting/pod_restart.json | 86 ++ .../grafana/oauth2/alerting/pod_start.json | 80 ++ .../grafana/oauth2/alerting/pvc_usage.json | 80 ++ .../share/grafana/oauth2/alerting/test.json | 74 -- .../grafana/provisioning/alerting/drycc.yaml | 806 +++++++++++++++++- 7 files changed, 1212 insertions(+), 75 deletions(-) create mode 100644 rootfs/usr/share/grafana/oauth2/alerting/pod_cpu.json create mode 100644 rootfs/usr/share/grafana/oauth2/alerting/pod_memory.json create mode 100644 rootfs/usr/share/grafana/oauth2/alerting/pod_restart.json create mode 100644 rootfs/usr/share/grafana/oauth2/alerting/pod_start.json create mode 100644 rootfs/usr/share/grafana/oauth2/alerting/pvc_usage.json delete mode 100644 rootfs/usr/share/grafana/oauth2/alerting/test.json diff --git a/rootfs/usr/share/grafana/oauth2/alerting/pod_cpu.json b/rootfs/usr/share/grafana/oauth2/alerting/pod_cpu.json new file mode 100644 index 0000000..4e0a480 --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/alerting/pod_cpu.json @@ -0,0 +1,80 @@ +{ + "uid": "pod_cpu", + "title": "pod cpu usage over 80%", + "condition": "C", + "data": [ + { + "refId": "A", + "relativeTimeRange": { + "from": 600, + "to": 0 + }, + "datasourceUid": "prometheus_on_drycc", + "model": { + "editorMode": "code", + "expr": "(sum(rate(container_cpu_usage_seconds_total{image!=\"\"}[2m])) by (pod,namespace)/sum(kube_pod_container_resource_limits{resource=\"cpu\"}) by (pod,namespace) * 100) \u003e 80", + "instant": true, + "intervalMs": 1000, + "legendFormat": "__auto", + "maxDataPoints": 43200, + "range": false, + "refId": "A" + } + }, + { + "refId": "C", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [ + 0 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "C" + ] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "A", + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": "C", + "type": "threshold" + } + } + ], + "noDataState": "OK", + "execErrState": "KeepLast", + "for": "2m", + "ruleGroup": "middle", + "annotations": { + "description": "namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} 's cpu usage over {{$value }}%.", + "summary": "CPU Usage of Pod limit on has exceeded 80%." + }, + "isPaused": false, + "notification_settings": { + "receiver": "grafana-default-email" + }, + "folderUID": "drycc" +} diff --git a/rootfs/usr/share/grafana/oauth2/alerting/pod_memory.json b/rootfs/usr/share/grafana/oauth2/alerting/pod_memory.json new file mode 100644 index 0000000..3eead7f --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/alerting/pod_memory.json @@ -0,0 +1,81 @@ +{ + "uid": "pod_memory", + "title": "pod memory usage over 80%", + "condition": "C", + "data": [ + { + "refId": "A", + "relativeTimeRange": { + "from": 600, + "to": 0 + }, + "datasourceUid": "prometheus_on_drycc", + "model": { + "editorMode": "code", + "expr": "(sum(container_memory_working_set_bytes{pod!=\"\",container!=\"\"}) by (pod, namespace)) / (sum(container_spec_memory_limit_bytes{pod!=\"\"}) by (pod, namespace)) \u003e 0.8 and (sum(container_memory_working_set_bytes{pod!=\"\",container!=\"\"}) by (pod, namespace)) / (sum (container_spec_memory_limit_bytes{pod!=\"\"}) by (pod, namespace)) \u003c 2", + "instant": true, + "intervalMs": 1000, + "legendFormat": "__auto", + "maxDataPoints": 43200, + "range": false, + "refId": "A" + } + }, + { + "refId": "C", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [ + 0 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "C" + ] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "A", + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": "C", + "type": "threshold" + } + } + ], + "noDataState": "OK", + "execErrState": "KeepLast", + "for": "2m", + "ruleGroup": "middle", + "keepFiringFor": "2m", + "annotations": { + "description": "namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} 's memory usage over {{ $value }}%.", + "summary": "Memory Usage of Pod limit on has exceeded 80%." + }, + "isPaused": false, + "notification_settings": { + "receiver": "grafana-default-email" + }, + "folderUID": "drycc" +} diff --git a/rootfs/usr/share/grafana/oauth2/alerting/pod_restart.json b/rootfs/usr/share/grafana/oauth2/alerting/pod_restart.json new file mode 100644 index 0000000..ae9c144 --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/alerting/pod_restart.json @@ -0,0 +1,86 @@ +{ + "uid": "pod_restart", + "title": "pod restart", + "annotations": { + "description": "namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} restart.", + "summary": "Pod restart alert" + }, + "condition": "C", + "data": [ + { + "datasourceUid": "prometheus_on_drycc", + "model": { + "editorMode": "code", + "expr": "increase(kube_pod_container_status_restarts_total{pod!~\".*job.*\"}[30m]) > 0", + "instant": true, + "intervalMs": 1000, + "legendFormat": "__auto", + "maxDataPoints": 43200, + "range": false, + "refId": "A" + }, + "queryType": "", + "refId": "A", + "relativeTimeRange": { + "from": 600, + "to": 0 + } + }, + { + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [ + 0 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "C" + ] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "A", + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": "C", + "type": "threshold" + }, + "queryType": "", + "refId": "C", + "relativeTimeRange": { + "from": 0, + "to": 0 + } + } + ], + "execErrState": "OK", + "folderUID": "drycc", + "id": 1, + "isPaused": false, + "keep_firing_for": "2m", + "noDataState": "OK", + "notification_settings": { + "receiver": "grafana-default-email" + }, + "orgID": 1, + "record": null, + "ruleGroup": "middle", + "for": "2m" +} diff --git a/rootfs/usr/share/grafana/oauth2/alerting/pod_start.json b/rootfs/usr/share/grafana/oauth2/alerting/pod_start.json new file mode 100644 index 0000000..3fce527 --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/alerting/pod_start.json @@ -0,0 +1,80 @@ +{ + "uid": "pod_start", + "title": "pod start fail", + "condition": "C", + "data": [ + { + "refId": "A", + "relativeTimeRange": { + "from": 600, + "to": 0 + }, + "datasourceUid": "prometheus_on_drycc", + "model": { + "editorMode": "code", + "expr": "kube_pod_status_phase{pod!~\".*job.*\", phase=~\"Failed|Unknown\"} == 1", + "instant": true, + "intervalMs": 1000, + "legendFormat": "__auto", + "maxDataPoints": 43200, + "range": false, + "refId": "A" + } + }, + { + "refId": "C", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [ + 0 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "C" + ] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "A", + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": "C", + "type": "threshold" + } + } + ], + "noDataState": "OK", + "execErrState": "KeepLast", + "for": "2m", + "ruleGroup": "middle", + "annotations": { + "description": "namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} start fail.", + "summary": "Pod start fail" + }, + "isPaused": false, + "notification_settings": { + "receiver": "grafana-default-email" + }, + "folderUID": "drycc" +} diff --git a/rootfs/usr/share/grafana/oauth2/alerting/pvc_usage.json b/rootfs/usr/share/grafana/oauth2/alerting/pvc_usage.json new file mode 100644 index 0000000..9aee76a --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/alerting/pvc_usage.json @@ -0,0 +1,80 @@ +{ + "uid": "pvc_usage", + "title": "pvc usage over 80%", + "condition": "C", + "data": [ + { + "refId": "A", + "relativeTimeRange": { + "from": 600, + "to": 0 + }, + "datasourceUid": "prometheus_on_drycc", + "model": { + "editorMode": "code", + "expr": "kubelet_volume_stats_used_bytes{} / kubelet_volume_stats_capacity_bytes{} \u003e 0.8", + "instant": true, + "intervalMs": 1000, + "legendFormat": "__auto", + "maxDataPoints": 43200, + "range": false, + "refId": "A" + } + }, + { + "refId": "C", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [ + 0 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "C" + ] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "A", + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": "C", + "type": "threshold" + } + } + ], + "noDataState": "OK", + "execErrState": "KeepLast", + "for": "2m", + "annotations": { + "description": "pvc capacity {{$labels.namespace}} {{$labels.persistentvolumeclaim}} on {{$labels.topology_topolvm_io_node}} usage over {{$value }}%.", + "summary": "Pvc usage over 80%" + }, + "isPaused": false, + "notification_settings": { + "receiver": "grafana-default-email" + }, + "folderUID": "drycc", + "ruleGroup": "middle" +} diff --git a/rootfs/usr/share/grafana/oauth2/alerting/test.json b/rootfs/usr/share/grafana/oauth2/alerting/test.json deleted file mode 100644 index 0708db4..0000000 --- a/rootfs/usr/share/grafana/oauth2/alerting/test.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "title": "TEST-API_1", - "ruleGroup": "API", - "folderUID": "SET_FOLDER_UID", - "noDataState": "OK", - "execErrState": "OK", - "for": "5m", - "orgId": 1, - "uid": "", - "condition": "B", - "annotations": { - "summary": "test_api_1" - }, - "labels": { - "API": "test1" - }, - "data": [ - { - "refId": "A", - "queryType": "", - "relativeTimeRange": { - "from": 600, - "to": 0 - }, - "datasourceUid": "XXXXXXXXX-XXXXXXXXX-XXXXXXXXXX", - "model": { - "expr": "up", - "hide": false, - "intervalMs": 1000, - "maxDataPoints": 43200, - "refId": "A" - } - }, - { - "refId": "B", - "queryType": "", - "relativeTimeRange": { - "from": 0, - "to": 0 - }, - "datasourceUid": "-100", - "model": { - "conditions": [ - { - "evaluator": { - "params": [6], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "params": ["A"] - }, - "reducer": { - "params": [], - "type": "last" - }, - "type": "query" - } - ], - "datasource": { - "type": "__expr__", - "uid": "-100" - }, - "hide": false, - "intervalMs": 1000, - "maxDataPoints": 43200, - "refId": "B", - "type": "classic_conditions" - } - } - ] -} \ No newline at end of file diff --git a/rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml b/rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml index eecb3a0..047f005 100644 --- a/rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml +++ b/rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml @@ -1 +1,805 @@ -apiVersion: 1 \ No newline at end of file +apiVersion: 1 +groups: + - orgId: 1 + name: middle + folder: drycc + interval: 2m + rules: + - uid: eeq3o5l9asa2od + editable: true + title: pod restart + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: increase(kube_pod_container_status_restarts_total{pod!~".*job.*"}[30m]) > 0 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + keepFiringFor: 2m + annotations: + description: 'namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} restart.' + summary: Pod restart alert + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: deq6i5dvnlkw0e + editable: true + title: pod cpu usage over 80% + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: (sum(rate(container_cpu_usage_seconds_total{image!=""}[2m])) by (pod,namespace)/sum(kube_pod_container_resource_limits{resource="cpu"}) by (pod,namespace) * 100) > 80 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: 'namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} ''s cpu usage over {{$value }}%.' + summary: CPU Usage of Pod limit on has exceeded 80%. + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: beq6l7fts9nnke + title: pod memory usage over 80% + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: (sum(container_memory_working_set_bytes{pod!="",container!=""}) by (pod, namespace)) / (sum(container_spec_memory_limit_bytes{pod!=""}) by (pod, namespace)) > 0.8 and (sum(container_memory_working_set_bytes{pod!="",container!=""}) by (pod, namespace)) / (sum (container_spec_memory_limit_bytes{pod!=""}) by (pod, namespace)) < 2 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + keepFiringFor: 2m + annotations: + description: 'namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} ''s memory usage over {{ $value }}%.' + summary: Memory Usage of Pod limit on has exceeded 80%. + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: eeq6lm6biyfb4e + title: pod start fail + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: kube_pod_status_phase{pod!~".*job.*", phase=~"Failed|Unknown"} == 1 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: 'namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} start fail.' + summary: Pod start fail + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: feq6m6yr27togc + title: job exec error + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: kube_job_status_failed{job="drycc-kube-state-metrics"} == 1 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: 'namespace: {{ $labels.namespace }}, job: {{ $labels.job_name }} exec {{ $labels.reason }}.' + summary: Job exec error + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: beqhp6i29jx8gb + title: pvc usage over 80% + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: kubelet_volume_stats_used_bytes{job="kubernetes-nodes"} / kubelet_volume_stats_capacity_bytes{job="kubernetes-nodes"} > 0.8 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: pvc capacity {{$labels.namespace}} {{$labels.persistentvolumeclaim}} on {{$labels.topology_topolvm_io_node}} usage over {{$value }}%. + summary: Pvc usage over 80% + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: deqhpblphb7k0c + title: mysql down + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: mysql_up != 1 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: MySQL {{$labels.namespace}} {{$labels.pod}} on {{$labels.node}} is not up. + summary: MySQL not up. + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: feqhpe2zz56o0d + title: mysql group replcation down + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: mysql_perf_schema_replication_group_member_info{member_state!="ONLINE"} == 1 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: MySQL group replication {{$labels.namespace}} {{$labels.pod}} on {{$labels.node}} is not up. + summary: MySQL group replication not up. + isPaused: false + notification_settings: + receiver: grafana-default-email + - orgId: 1 + name: high + folder: drycc + interval: 1m + rules: + - uid: beq6ljd99ukg0c + title: node down + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: up{job="drycc-node-exporter"} == 0 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 1m + annotations: + description: 'node: {{ $labels.instance }} down.' + summary: Node down + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: eeq6lu5wz3eo0c + title: cluster node over pressure + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: kube_node_status_condition{condition=~"OutOfDisk|MemoryPressure|DiskPressure",status!="false"} == 1 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 1m + annotations: + description: 'node: {{ $labels.node }} has {{ $labels.condition }}.' + summary: Cluster node over pressure + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: deq6lwic91slce + title: cluster node status error + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: kube_node_status_condition{condition="Ready",status!="true"} == 1 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 1m + annotations: + description: 'node: {{ $labels.node }} status error.' + summary: Cluster node status error + isPaused: false + notification_settings: + receiver: grafana-default-email + - orgId: 1 + name: middle + folder: drycc + interval: 2m + rules: + - uid: beq6ptjdxwmpsc + title: node cpu usage over 80% + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: (1 - avg by (instance) (irate(node_cpu_seconds_total{job="drycc-node-exporter",mode="idle"}[5m]))) * 100 > 80 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: 'instance: {{ $labels.instance }} cpu usage over {{ $value }}%' + summary: Node cpu high over 80 + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: ceqhohsqqyr5sb + title: node memory usage over 80% + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: (1-(node_memory_MemAvailable_bytes{job="drycc-node-exporter"} / (node_memory_MemTotal_bytes{job="drycc-node-exporter"})))* 100 > 80 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: 'instance: {{ $labels.instance }} memory usage over {{ $value }}%.' + summary: Node memory over 80 + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: aeqhp0wbcsb9cf + title: node systen disk usage over 80% + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: (1 - node_filesystem_avail_bytes{job="drycc-node-exporter",fstype=~"ext4|xfs"} / node_filesystem_size_bytes{job="drycc-node-exporter",fstype=~"ext4|xfs"}) * 100 > 80 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: 'instance: {{ $labels.instance }} disk usage over {{ $value }}%.' + summary: Node systen disk usage over 80 + isPaused: false + notification_settings: + receiver: grafana-default-email + - orgId: 1 + name: slow + folder: drycc + interval: 1d + rules: + - uid: feq6lfrsmsum8a + title: node certificate expiration + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: ((x509_cert_not_after - time()) / 86400) < 15 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + annotations: + description: 'node: {{ $labels.node}} cert: {{ $labels.filepath }} will expire in {{ $labels.value }} days.' + summary: Node certificate expiration + isPaused: false + notification_settings: + receiver: grafana-default-email From e2a7567f0c8e9fdec81887c3a1f4f0b6e2eee223 Mon Sep 17 00:00:00 2001 From: Eamon Date: Fri, 18 Jul 2025 14:06:41 +0800 Subject: [PATCH 37/50] chore(grafana): add drycc dashborads (#40) * chore(grafana): add dashborads * chore(grafana): rename istio to gateway --------- Co-authored-by: zhangeamon --- .../grafana/templates/grafana-deployment.yaml | 2 +- rootfs/Dockerfile | 2 +- .../oauth2/dashboards/drycc_applicaion.json | 1147 +++++++++++ .../dashboards/drycc_application_logs.json | 27 +- .../oauth2/dashboards/drycc_gateway.json | 1677 +++++++++++++++++ .../oauth2/dashboards/drycc_instance.json | 1193 ++++++++++++ .../oauth2/dashboards/drycc_volume.json | 454 +++++ .../usr/share/grafana/oauth2/hook/__init__.py | 6 +- .../usr/share/grafana/oauth2/hook/grafana.py | 14 +- .../grafana/provisioning/alerting/drycc.yaml | 1604 ++++++++-------- .../provisioning/dashboards/drycc.yaml | 5 +- 11 files changed, 5296 insertions(+), 835 deletions(-) create mode 100644 rootfs/usr/share/grafana/oauth2/dashboards/drycc_applicaion.json create mode 100644 rootfs/usr/share/grafana/oauth2/dashboards/drycc_gateway.json create mode 100644 rootfs/usr/share/grafana/oauth2/dashboards/drycc_instance.json create mode 100644 rootfs/usr/share/grafana/oauth2/dashboards/drycc_volume.json diff --git a/charts/grafana/templates/grafana-deployment.yaml b/charts/grafana/templates/grafana-deployment.yaml index 93a4a45..7e653c5 100644 --- a/charts/grafana/templates/grafana-deployment.yaml +++ b/charts/grafana/templates/grafana-deployment.yaml @@ -37,7 +37,7 @@ spec: - netcat - -v - -u - - $(GF_DATABASE_URL) + - $(GF_DATABASE_URL),http://drycc-controller-api {{- include "grafana.envs" . | indent 8 }} containers: - name: drycc-grafana diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 21ce9a7..cbac2e3 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -4,7 +4,7 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/usr/share/grafana \ - GRAFANA_VERSION="12.0.2" \ + GRAFANA_VERSION="12.1.0" \ JQ_VERSION="1.7.1" \ CADDY_VERSION="2.9.1" \ PYTHON_VERSION="3.13" diff --git a/rootfs/usr/share/grafana/oauth2/dashboards/drycc_applicaion.json b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_applicaion.json new file mode 100644 index 0000000..bee7b27 --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_applicaion.json @@ -0,0 +1,1147 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "General Drycc App metrics.", + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 19, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 35, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "quickwit-quickwit-datasource", + "uid": "application_logs" + }, + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 19, + "x": 0, + "y": 1 + }, + "id": 34, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": true, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "quickwit-quickwit-datasource", + "uid": "application_logs" + }, + "metrics": [ + { + "id": "1", + "settings": {}, + "type": "logs" + } + ], + "query": "kubernetes.namespace_name:${app:singlequote} kubernetes.container_name:drycc-controller", + "refId": "A", + "timeField": "" + } + ], + "title": "", + "type": "logs" + }, + { + "datasource": { + "type": "quickwit-quickwit-datasource", + "uid": "application_logs" + }, + "description": "[Query Language](https://quickwit.io/docs/reference/query-language)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 19, + "y": 1 + }, + "id": 36, + "interval": "1m", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "quickwit-quickwit-datasource", + "uid": "application_logs" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "kubernetes.namespace_name:${app:singlequote} kubernetes.container_name:drycc-controller", + "refId": "A", + "timeField": "" + } + ], + "title": "Logs volume", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 8, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 19, + "x": 0, + "y": 10 + }, + "id": 15, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=~\"$app\",container!=\"\"}[5m])) by (pod) \n/\n(\n sum(container_spec_cpu_quota{namespace=~\"$app\",container!=\"\"}) by (pod) / 10000000\n)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Load 5m", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 19, + "y": 10 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum((container_spec_cpu_quota{namespace=~\"$app\",container!=\"\"}/1000)/100)", + "instant": true, + "legendFormat": "Total CPU Cores", + "range": false, + "refId": "A" + } + ], + "title": "Total CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "(increase(container_cpu_cfs_throttled_periods_total{namespace=~\"$app\",container!=\"\" }[5m]))/\n(increase(container_cpu_cfs_periods_total{namespace=~\"$app\",container!=\"\"}[5m]))", + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "CPUThrottling ", + "type": "timeseries" + } + ], + "title": "CPU", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 11, + "panels": [], + "title": "Memory", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 19, + "x": 0, + "y": 11 + }, + "id": 14, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "(container_memory_working_set_bytes{namespace=~\"$app\",container!=\"\"}) \n", + "hide": false, + "interval": "", + "legendFormat": "Used Memory", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PF6837BEDE254B550" + }, + "editorMode": "code", + "expr": "", + "hide": true, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "B" + } + ], + "title": "Used Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 19, + "y": 11 + }, + "id": 33, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(container_spec_memory_limit_bytes{namespace=~\"$app\",container!=\"\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Memory", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 2, + "panels": [], + "title": "Networking", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(container_network_receive_bytes_total{namespace=~\"$app\"}[5m])) ", + "hide": false, + "interval": "", + "legendFormat": "Total", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PF6837BEDE254B550" + }, + "editorMode": "code", + "expr": "sum(rate(container_network_receive_bytes_total{namespace=~\"$app\"}[5m])) by (pod)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=~\"$app\"}[5m]))", + "interval": "", + "legendFormat": "Total", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PF6837BEDE254B550" + }, + "editorMode": "code", + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=~\"$app\"}[5m])) by (pod)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Transmitted", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 27, + "panels": [], + "title": "Volume", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 28, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$app\",container!=\"\"}[1m]))\n", + "hide": false, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PF6837BEDE254B550" + }, + "editorMode": "code", + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$app\",container!=\"\"}[1m])) by(pod)", + "hide": false, + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "IOPS Read", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$app\",container!=\"\"}[1m]))\n", + "legendFormat": "Total", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$app\",container!=\"\"}[1m])) by(pod)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "IOPS Write", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "60s", + "schemaVersion": 41, + "tags": ["drycc"], + "templating": { + "list": [ + { + "current": { + "text": "demo1", + "value": "demo1" + }, + "definition": "label_values(namespace)", + "includeAll": false, + "label": "app", + "name": "app", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Drycc App", + "uid": "cerzlzfbcevapp", + "version": 9 +} diff --git a/rootfs/usr/share/grafana/oauth2/dashboards/drycc_application_logs.json b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_application_logs.json index 1d4d8ea..c34b157 100644 --- a/rootfs/usr/share/grafana/oauth2/dashboards/drycc_application_logs.json +++ b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_application_logs.json @@ -21,27 +21,12 @@ } ] }, - "description": "Application Logs. See also the Explore tab for ad-hoc log queries.", + "description": "General Drycc Logs metrics.", "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": 61, - "links": [ - { - "asDropdown": false, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "drycc" - ], - "targetBlank": false, - "title": "Drycc", - "tooltip": "", - "type": "dashboards", - "url": "" - } - ], + "links": [], "panels": [ { "datasource": { @@ -149,7 +134,7 @@ "type": "count" } ], - "query": "kubernetes.namespace_name:${app:singlequote} kubernetes.pod_name:${instance:singlequote} ${query:text}", + "query": "kubernetes.namespace_name:${app:singlequote} kubernetes.pod_name:${instance:singlequote} ${query:text} -kubernetes.container_name:drycc-controller", "refId": "A", "timeField": "" } @@ -210,7 +195,7 @@ "type": "logs" } ], - "query": "kubernetes.namespace_name:${app:singlequote} kubernetes.pod_name:${instance:singlequote} ${query:text}", + "query": "kubernetes.namespace_name:${app:singlequote} kubernetes.pod_name:${instance:singlequote} ${query:text} -kubernetes.container_name:drycc-controller", "refId": "A", "timeField": "" } @@ -222,7 +207,7 @@ "preload": false, "refresh": "", "schemaVersion": 41, - "tags": [], + "tags": ["drycc"], "templating": { "list": [ { @@ -315,4 +300,4 @@ "title": "Drycc Logs", "uid": "aepz8ewtzq4g0f", "version": 20 -} \ No newline at end of file +} diff --git a/rootfs/usr/share/grafana/oauth2/dashboards/drycc_gateway.json b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_gateway.json new file mode 100644 index 0000000..82152e0 --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_gateway.json @@ -0,0 +1,1677 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "General Drycc Gateway metrics.", + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 79, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 106, + "panels": [], + "title": "General", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 12, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "round(sum(irate(istio_requests_total{destination_service=~\"$service\"}[$__rate_interval])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": "Client Request Volume", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red" + }, + { + "color": "dark-yellow", + "value": 0.95 + }, + { + "color": "dark-green", + "value": 0.99 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 14, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(irate(istio_requests_total{reporter=~\"$qrep\",destination_service=~\"$service\",response_code!~\"5.*\"}[$__rate_interval])) / (sum(irate(istio_requests_total{reporter=~\"$qrep\",destination_service=~\"$service\"}[$__rate_interval])) or on () vector(1))", + "format": "time_series", + "intervalFactor": 1, + "range": true, + "refId": "A" + } + ], + "title": "Client Success Rate (non-5xx responses)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 87, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\",destination_service=~\"$service\"}[$__rate_interval])) by (le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\",destination_service=~\"$service\"}[$__rate_interval])) by (le))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "P50", + "refId": "A" + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\",destination_service=~\"$service\"}[$__rate_interval])) by (le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\",destination_service=~\"$service\"}[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P90", + "refId": "B" + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\",destination_service=~\"$service\"}[$__rate_interval])) by (le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\",destination_service=~\"$service\"}[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P99", + "refId": "C" + } + ], + "title": "Client Request Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 95 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 99 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 5 + }, + "id": 98, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(irate(istio_requests_total{reporter=\"destination\",destination_service=~\"$service\",response_code!~\"5.*\"}[$__rate_interval])) / (sum(irate(istio_requests_total{reporter=\"destination\",destination_service=~\"$service\"}[$__rate_interval])) or on () vector(1))", + "format": "time_series", + "intervalFactor": 1, + "range": true, + "refId": "A" + } + ], + "title": "Server Success Rate (non-5xx responses)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 5 + }, + "id": 100, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "expr": "sum(irate(istio_tcp_sent_bytes_total{reporter=~\"$qrep\", destination_service=~\"$service\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "TCP Sent Bytes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 5 + }, + "id": 84, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "expr": "sum(irate(istio_tcp_received_bytes_total{reporter=~\"$qrep\", destination_service=~\"$service\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "TCP Received Bytes", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 104, + "panels": [], + "title": "Client Workloads", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\",destination_service=~\"$service\",reporter=~\"$qrep\"}[$__rate_interval])) by (source_workload, source_workload_namespace,response_code), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} : {{ response_code }} ", + "range": true, + "refId": "A", + "step": 2 + }, + { + "editorMode": "code", + "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", reporter=~\"$qrep\"}[$__rate_interval])) by (source_workload, source_workload_namespace, response_code), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} : {{ response_code }}", + "range": true, + "refId": "B", + "step": 2 + } + ], + "title": "Incoming Requests By Source And Response Code", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.01, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(irate(istio_requests_total{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\"}[$__rate_interval])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} (🔐mTLS)", + "range": true, + "refId": "A", + "step": 2 + }, + { + "editorMode": "code", + "expr": "sum(irate(istio_requests_total{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\", }[$__rate_interval])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }}", + "range": true, + "refId": "B", + "step": 2 + } + ], + "title": "Incoming Success Rate (non-5xx responses) By Source", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", + "range": true, + "refId": "A", + "step": 2 + }, + { + "editorMode": "code", + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", + "range": true, + "refId": "B", + "step": 2 + }, + { + "editorMode": "code", + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", + "range": true, + "refId": "C", + "step": 2 + }, + { + "editorMode": "code", + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", + "range": true, + "refId": "D", + "step": 2 + }, + { + "editorMode": "code", + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", + "range": true, + "refId": "E", + "step": 2 + }, + { + "editorMode": "code", + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", + "range": true, + "refId": "F", + "step": 2 + }, + { + "editorMode": "code", + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", + "range": true, + "refId": "G", + "step": 2 + }, + { + "editorMode": "code", + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", + "range": true, + "refId": "H", + "step": 2 + } + ], + "title": "Incoming Request Duration By Source", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", + "range": true, + "refId": "A", + "step": 2 + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", + "range": true, + "refId": "B", + "step": 2 + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", + "range": true, + "refId": "C", + "step": 2 + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", + "range": true, + "refId": "D", + "step": 2 + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", + "range": true, + "refId": "E", + "step": 2 + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", + "range": true, + "refId": "F", + "step": 2 + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", + "range": true, + "refId": "G", + "step": 2 + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", + "range": true, + "refId": "H", + "step": 2 + } + ], + "title": "Incoming Request Size By Source", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", + "range": true, + "refId": "A", + "step": 2 + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", + "range": true, + "refId": "B", + "step": 2 + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", + "range": true, + "refId": "C", + "step": 2 + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", + "range": true, + "refId": "D", + "step": 2 + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", + "range": true, + "refId": "E", + "step": 2 + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", + "range": true, + "refId": "F", + "step": 2 + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", + "range": true, + "refId": "G", + "step": 2 + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", + "range": true, + "refId": "H", + "step": 2 + } + ], + "title": "Response Size By Source", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 80, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", + "range": true, + "refId": "A", + "step": 2 + }, + { + "editorMode": "code", + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", + "range": true, + "refId": "B", + "step": 2 + } + ], + "title": "Bytes Received from Incoming TCP Connection", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 82, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy=\"mutual_tls\", reporter=~\"$qrep\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", + "range": true, + "refId": "A", + "step": 2 + }, + { + "editorMode": "code", + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy!=\"mutual_tls\", reporter=~\"$qrep\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", + "range": true, + "refId": "B", + "step": 2 + } + ], + "title": "Bytes Sent to Incoming TCP Connection", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "1m", + "schemaVersion": 41, + "tags": ["drycc"], + "templating": { + "list": [ + { + "definition": "label_values(istio_requests_total,namespace)", + "description": "namespace ", + "label": "app", + "name": "app", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(istio_requests_total,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allowCustomValue": false, + "definition": "label_values(istio_requests_total{namespace=\"$app\"},pod)", + "description": "pod ", + "label": "instance", + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(istio_requests_total{namespace=\"$app\"},pod)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "", + "allowCustomValue": false, + "definition": "query_result(sum(istio_requests_total{namespace=\"$app\",pod=\"$instance\"}) by (destination_service) or sum(istio_tcp_sent_bytes_total{namespace=\"$app\",pod=\"$instance\"}) by (destination_service))", + "includeAll": true, + "label": "Service", + "multi": true, + "name": "service", + "options": [], + "query": { + "qryType": 3, + "query": "query_result(sum(istio_requests_total{namespace=\"$app\",pod=\"$instance\"}) by (destination_service) or sum(istio_tcp_sent_bytes_total{namespace=\"$app\",pod=\"$instance\"}) by (destination_service))", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/.*destination_service=\"([^\"]*).*/", + "type": "query" + }, + { + "allowCustomValue": false, + "current": { + "text": "source", + "value": "source" + }, + "hide": 2, + "includeAll": false, + "label": "Reporter", + "name": "qrep", + "options": [ + { + "selected": true, + "text": "source", + "value": "source" + } + ], + "query": "source", + "type": "custom" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": ["5m", "15m", "30m", "1h", "2h", "1d"] + }, + "timezone": "", + "title": "Drycc Gateway", + "uid": "bb2f5242-9429-4f60-b520-20b7c70a0a9a", + "version": 12 +} diff --git a/rootfs/usr/share/grafana/oauth2/dashboards/drycc_instance.json b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_instance.json new file mode 100644 index 0000000..c27bb01 --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_instance.json @@ -0,0 +1,1193 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "General Drycc Instance metrics.", + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 69, + "links": [], + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 8, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 20, + "x": 0, + "y": 1 + }, + "id": 15, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}[10m])) by (pod) \n/\n(\n sum(container_spec_cpu_quota{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}) by (pod) / 10000000\n)", + "hide": false, + "interval": "", + "legendFormat": "10 minute load average", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}[5m])) by (pod) \n/\n(\n sum(container_spec_cpu_quota{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}) by (pod) / 10000000\n)", + "hide": false, + "instant": false, + "legendFormat": "5 minute load average", + "range": true, + "refId": "B" + } + ], + "title": "CPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "(container_spec_cpu_quota{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}/1000)/100", + "instant": true, + "legendFormat": "Total CPU Cores", + "range": false, + "refId": "A" + } + ], + "title": "Total CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "(increase(container_cpu_cfs_throttled_periods_total{namespace=~\"$app\",container!=\"\",pod=~\"$instance\" }[5m]))\n /\n(increase(container_cpu_cfs_periods_total{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}[5m]))", + "instant": false, + "legendFormat": "throtting", + "range": true, + "refId": "A" + } + ], + "title": "CPUThrottling ", + "type": "timeseries" + } + ], + "title": "CPU", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 11, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 143 + }, + "id": 9, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Name", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(container_memory_rss{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}) by (pod)", + "hide": false, + "interval": "", + "legendFormat": "rss", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(container_memory_cache{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}) by (pod)", + "hide": false, + "instant": false, + "legendFormat": "cache", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "container_spec_memory_limit_bytes{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "total", + "range": true, + "refId": "C" + } + ], + "title": "Used Memory\t", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 143 + }, + "id": 14, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "(container_memory_working_set_bytes{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}*100) /container_spec_memory_limit_bytes{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}\n", + "hide": false, + "interval": "", + "legendFormat": "Used Memory", + "range": true, + "refId": "A" + } + ], + "title": "Used Memory", + "type": "timeseries" + } + ], + "title": "Memory", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 2, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(container_network_receive_bytes_total{namespace=~\"$app\",pod=~\"$instance\"}[5m])) by (pod)", + "hide": false, + "interval": "", + "legendFormat": "received", + "range": true, + "refId": "A" + } + ], + "title": "Received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 77 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=~\"$app\",pod=~\"$instance\"}[5m])) by (pod)", + "interval": "", + "legendFormat": "Transmitted", + "range": true, + "refId": "A" + } + ], + "title": "Transmitted", + "type": "timeseries" + } + ], + "title": "Networking", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 23, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 280 + }, + "id": 24, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "container_processes{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}", + "instant": false, + "legendFormat": "Process Count", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "container_threads{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Thread Count", + "range": true, + "refId": "B" + } + ], + "title": "", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 280 + }, + "id": 26, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "container_file_descriptors{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}\n", + "instant": false, + "legendFormat": "File Descriptors\t", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "container_sockets{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Sockets", + "range": true, + "refId": "B" + } + ], + "title": "", + "type": "timeseries" + } + ], + "title": "Process", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 27, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 28, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}[1m]))\n", + "hide": false, + "instant": false, + "legendFormat": "read", + "range": true, + "refId": "A" + } + ], + "title": "IOPS Read", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 30, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "rate(container_fs_writes_bytes_total{namespace=~\"$app\",container!=\"\",pod=~\"$instance\"}[1m])\n", + "legendFormat": "write", + "range": true, + "refId": "A" + } + ], + "title": "IOPS Write", + "type": "timeseries" + } + ], + "title": "TmpDisk", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": ["drycc"], + "templating": { + "list": [ + { + "definition": "label_values(namespace)", + "includeAll": false, + "label": "app", + "name": "app", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allowCustomValue": false, + + "definition": "label_values({namespace=\"$app\"},pod)", + "includeAll": false, + "label": "instance", + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({namespace=\"$app\"},pod)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Drycc Instances", + "uid": "cerzlzfbcevpcc", + "version": 18 +} diff --git a/rootfs/usr/share/grafana/oauth2/dashboards/drycc_volume.json b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_volume.json new file mode 100644 index 0000000..d044890 --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_volume.json @@ -0,0 +1,454 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "General Drycc Volume metrics.", + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 74, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 18, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max", "min"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Space", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum without(instance, node) (kubelet_volume_stats_available_bytes{job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "range": true, + "refId": "B" + } + ], + "title": "Volume Space Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 3, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "(\n kubelet_volume_stats_capacity_bytes{job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{ job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Volume Space Usage", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 18, + "x": 0, + "y": 7 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max", "min"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum without(instance, node) (kubelet_volume_stats_inodes_used{ job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used inodes", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_inodes{job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_inodes_used{job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "range": true, + "refId": "B" + } + ], + "title": "Volume inodes Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_on_drycc" + }, + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 5, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "kubelet_volume_stats_inodes_used{job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Volume inodes Usage", + "type": "stat" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": ["drycc"], + "templating": { + "list": [ + { + "current": { + "text": "", + "value": "" + }, + + "definition": "", + "hide": 2, + "includeAll": false, + "label": "cluster", + "name": "cluster", + "options": [], + "query": "", + "refresh": 2, + "regex": "", + "type": "query" + }, + { + "definition": "label_values(kubelet_volume_stats_available_bytes,namespace)", + "label": "Namespace", + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kubelet_volume_stats_available_bytes,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "definition": "label_values(kubelet_volume_stats_available_bytes{namespace=\"$namespace\"},persistentvolumeclaim)", + "description": "", + "includeAll": false, + "label": "PersistentVolumeClaim", + "name": "volume", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kubelet_volume_stats_available_bytes{namespace=\"$namespace\"},persistentvolumeclaim)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Drycc Volume", + "uid": "ab92a8e8041bd567a", + "version": 3 +} diff --git a/rootfs/usr/share/grafana/oauth2/hook/__init__.py b/rootfs/usr/share/grafana/oauth2/hook/__init__.py index 4651a65..89233ba 100644 --- a/rootfs/usr/share/grafana/oauth2/hook/__init__.py +++ b/rootfs/usr/share/grafana/oauth2/hook/__init__.py @@ -1,7 +1,9 @@ from .grafana import ( - init_org, sync_user, sync_role, sync_datasources, sync_alerting, sync_dashboards + init_org, sync_user, sync_role, sync_datasources, sync_folder, sync_dashboards, sync_alerting ) startup_hooks = [init_org] -login_hooks = [sync_user, sync_role, sync_datasources, sync_alerting, sync_dashboards] +login_hooks = [ + sync_user, sync_role, sync_datasources, sync_folder, sync_dashboards, sync_alerting +] destroy_hooks = [] diff --git a/rootfs/usr/share/grafana/oauth2/hook/grafana.py b/rootfs/usr/share/grafana/oauth2/hook/grafana.py index d13647e..e4943ec 100644 --- a/rootfs/usr/share/grafana/oauth2/hook/grafana.py +++ b/rootfs/usr/share/grafana/oauth2/hook/grafana.py @@ -82,6 +82,15 @@ async def sync_role(context: dict, token: dict, userinfo: dict): await init_org(org_id=context["org_id"], name=userinfo["preferred_username"]) +async def sync_folder(context: dict, token: dict, userinfo: dict): + async with httpx.AsyncClient() as client: + await client.post( + api_url("/api/folders"), + headers=api_headers(context, userinfo), + json={"uid": "drycc", "title": "drycc"}, + ) + + async def sync_alerting(context: dict, token: dict, userinfo: dict): alerting_path = os.path.join(os.path.dirname(__file__), "..", "alerting") async with httpx.AsyncClient() as client: @@ -132,11 +141,6 @@ async def sync_datasources(context: dict, token: dict, userinfo: dict): async def sync_dashboards(context: dict, token: dict, userinfo: dict): dashboards_path = os.path.join(os.path.dirname(__file__), "..", "dashboards") async with httpx.AsyncClient() as client: - await client.post( - api_url("/api/folders"), - headers=api_headers(context, userinfo), - json={"uid": "drycc", "title": "drycc"}, - ) for filename in os.listdir(dashboards_path): with open(os.path.join(dashboards_path, filename)) as f: dashboard = json.load(f) diff --git a/rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml b/rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml index 047f005..a4f1545 100644 --- a/rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml +++ b/rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml @@ -1,805 +1,803 @@ apiVersion: 1 groups: - - orgId: 1 - name: middle - folder: drycc - interval: 2m - rules: - - uid: eeq3o5l9asa2od - editable: true - title: pod restart - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus_on_drycc - model: - editorMode: code - expr: increase(kube_pod_container_status_restarts_total{pod!~".*job.*"}[30m]) > 0 - instant: true - intervalMs: 1000 - legendFormat: __auto - maxDataPoints: 43200 - range: false - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: OK - execErrState: KeepLast - for: 2m - keepFiringFor: 2m - annotations: - description: 'namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} restart.' - summary: Pod restart alert - isPaused: false - notification_settings: - receiver: grafana-default-email - - uid: deq6i5dvnlkw0e - editable: true - title: pod cpu usage over 80% - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus_on_drycc - model: - editorMode: code - expr: (sum(rate(container_cpu_usage_seconds_total{image!=""}[2m])) by (pod,namespace)/sum(kube_pod_container_resource_limits{resource="cpu"}) by (pod,namespace) * 100) > 80 - instant: true - intervalMs: 1000 - legendFormat: __auto - maxDataPoints: 43200 - range: false - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: OK - execErrState: KeepLast - for: 2m - annotations: - description: 'namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} ''s cpu usage over {{$value }}%.' - summary: CPU Usage of Pod limit on has exceeded 80%. - isPaused: false - notification_settings: - receiver: grafana-default-email - - uid: beq6l7fts9nnke - title: pod memory usage over 80% - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus_on_drycc - model: - editorMode: code - expr: (sum(container_memory_working_set_bytes{pod!="",container!=""}) by (pod, namespace)) / (sum(container_spec_memory_limit_bytes{pod!=""}) by (pod, namespace)) > 0.8 and (sum(container_memory_working_set_bytes{pod!="",container!=""}) by (pod, namespace)) / (sum (container_spec_memory_limit_bytes{pod!=""}) by (pod, namespace)) < 2 - instant: true - intervalMs: 1000 - legendFormat: __auto - maxDataPoints: 43200 - range: false - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: OK - execErrState: KeepLast - for: 2m - keepFiringFor: 2m - annotations: - description: 'namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} ''s memory usage over {{ $value }}%.' - summary: Memory Usage of Pod limit on has exceeded 80%. - isPaused: false - notification_settings: - receiver: grafana-default-email - - uid: eeq6lm6biyfb4e - title: pod start fail - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus_on_drycc - model: - editorMode: code - expr: kube_pod_status_phase{pod!~".*job.*", phase=~"Failed|Unknown"} == 1 - instant: true - intervalMs: 1000 - legendFormat: __auto - maxDataPoints: 43200 - range: false - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: OK - execErrState: KeepLast - for: 2m - annotations: - description: 'namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} start fail.' - summary: Pod start fail - isPaused: false - notification_settings: - receiver: grafana-default-email - - uid: feq6m6yr27togc - title: job exec error - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus_on_drycc - model: - editorMode: code - expr: kube_job_status_failed{job="drycc-kube-state-metrics"} == 1 - instant: true - intervalMs: 1000 - legendFormat: __auto - maxDataPoints: 43200 - range: false - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: OK - execErrState: KeepLast - for: 2m - annotations: - description: 'namespace: {{ $labels.namespace }}, job: {{ $labels.job_name }} exec {{ $labels.reason }}.' - summary: Job exec error - isPaused: false - notification_settings: - receiver: grafana-default-email - - uid: beqhp6i29jx8gb - title: pvc usage over 80% - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus_on_drycc - model: - editorMode: code - expr: kubelet_volume_stats_used_bytes{job="kubernetes-nodes"} / kubelet_volume_stats_capacity_bytes{job="kubernetes-nodes"} > 0.8 - instant: true - intervalMs: 1000 - legendFormat: __auto - maxDataPoints: 43200 - range: false - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: OK - execErrState: KeepLast - for: 2m - annotations: - description: pvc capacity {{$labels.namespace}} {{$labels.persistentvolumeclaim}} on {{$labels.topology_topolvm_io_node}} usage over {{$value }}%. - summary: Pvc usage over 80% - isPaused: false - notification_settings: - receiver: grafana-default-email - - uid: deqhpblphb7k0c - title: mysql down - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus_on_drycc - model: - editorMode: code - expr: mysql_up != 1 - instant: true - intervalMs: 1000 - legendFormat: __auto - maxDataPoints: 43200 - range: false - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: OK - execErrState: KeepLast - for: 2m - annotations: - description: MySQL {{$labels.namespace}} {{$labels.pod}} on {{$labels.node}} is not up. - summary: MySQL not up. - isPaused: false - notification_settings: - receiver: grafana-default-email - - uid: feqhpe2zz56o0d - title: mysql group replcation down - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus_on_drycc - model: - editorMode: code - expr: mysql_perf_schema_replication_group_member_info{member_state!="ONLINE"} == 1 - instant: true - intervalMs: 1000 - legendFormat: __auto - maxDataPoints: 43200 - range: false - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: OK - execErrState: KeepLast - for: 2m - annotations: - description: MySQL group replication {{$labels.namespace}} {{$labels.pod}} on {{$labels.node}} is not up. - summary: MySQL group replication not up. - isPaused: false - notification_settings: - receiver: grafana-default-email - - orgId: 1 - name: high - folder: drycc - interval: 1m - rules: - - uid: beq6ljd99ukg0c - title: node down - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus_on_drycc - model: - editorMode: code - expr: up{job="drycc-node-exporter"} == 0 - instant: true - intervalMs: 1000 - legendFormat: __auto - maxDataPoints: 43200 - range: false - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: OK - execErrState: KeepLast - for: 1m - annotations: - description: 'node: {{ $labels.instance }} down.' - summary: Node down - isPaused: false - notification_settings: - receiver: grafana-default-email - - uid: eeq6lu5wz3eo0c - title: cluster node over pressure - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus_on_drycc - model: - editorMode: code - expr: kube_node_status_condition{condition=~"OutOfDisk|MemoryPressure|DiskPressure",status!="false"} == 1 - instant: true - intervalMs: 1000 - legendFormat: __auto - maxDataPoints: 43200 - range: false - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: OK - execErrState: KeepLast - for: 1m - annotations: - description: 'node: {{ $labels.node }} has {{ $labels.condition }}.' - summary: Cluster node over pressure - isPaused: false - notification_settings: - receiver: grafana-default-email - - uid: deq6lwic91slce - title: cluster node status error - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus_on_drycc - model: - editorMode: code - expr: kube_node_status_condition{condition="Ready",status!="true"} == 1 - instant: true - intervalMs: 1000 - legendFormat: __auto - maxDataPoints: 43200 - range: false - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: OK - execErrState: KeepLast - for: 1m - annotations: - description: 'node: {{ $labels.node }} status error.' - summary: Cluster node status error - isPaused: false - notification_settings: - receiver: grafana-default-email - - orgId: 1 - name: middle - folder: drycc - interval: 2m - rules: - - uid: beq6ptjdxwmpsc - title: node cpu usage over 80% - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus_on_drycc - model: - editorMode: code - expr: (1 - avg by (instance) (irate(node_cpu_seconds_total{job="drycc-node-exporter",mode="idle"}[5m]))) * 100 > 80 - instant: true - intervalMs: 1000 - legendFormat: __auto - maxDataPoints: 43200 - range: false - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: OK - execErrState: KeepLast - for: 2m - annotations: - description: 'instance: {{ $labels.instance }} cpu usage over {{ $value }}%' - summary: Node cpu high over 80 - isPaused: false - notification_settings: - receiver: grafana-default-email - - uid: ceqhohsqqyr5sb - title: node memory usage over 80% - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus_on_drycc - model: - editorMode: code - expr: (1-(node_memory_MemAvailable_bytes{job="drycc-node-exporter"} / (node_memory_MemTotal_bytes{job="drycc-node-exporter"})))* 100 > 80 - instant: true - intervalMs: 1000 - legendFormat: __auto - maxDataPoints: 43200 - range: false - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: OK - execErrState: KeepLast - for: 2m - annotations: - description: 'instance: {{ $labels.instance }} memory usage over {{ $value }}%.' - summary: Node memory over 80 - isPaused: false - notification_settings: - receiver: grafana-default-email - - uid: aeqhp0wbcsb9cf - title: node systen disk usage over 80% - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus_on_drycc - model: - editorMode: code - expr: (1 - node_filesystem_avail_bytes{job="drycc-node-exporter",fstype=~"ext4|xfs"} / node_filesystem_size_bytes{job="drycc-node-exporter",fstype=~"ext4|xfs"}) * 100 > 80 - instant: true - intervalMs: 1000 - legendFormat: __auto - maxDataPoints: 43200 - range: false - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: OK - execErrState: KeepLast - for: 2m - annotations: - description: 'instance: {{ $labels.instance }} disk usage over {{ $value }}%.' - summary: Node systen disk usage over 80 - isPaused: false - notification_settings: - receiver: grafana-default-email - - orgId: 1 - name: slow - folder: drycc - interval: 1d - rules: - - uid: feq6lfrsmsum8a - title: node certificate expiration - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus_on_drycc - model: - editorMode: code - expr: ((x509_cert_not_after - time()) / 86400) < 15 - instant: true - intervalMs: 1000 - legendFormat: __auto - maxDataPoints: 43200 - range: false - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: OK - execErrState: KeepLast - annotations: - description: 'node: {{ $labels.node}} cert: {{ $labels.filepath }} will expire in {{ $labels.value }} days.' - summary: Node certificate expiration - isPaused: false - notification_settings: - receiver: grafana-default-email + - orgId: 1 + name: middle + folder: drycc + interval: 2m + rules: + - uid: eeq3o5l9asa2od + title: pod restart + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: increase(kube_pod_container_status_restarts_total{pod!~".*job.*"}[30m]) > 0 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + keepFiringFor: 2m + annotations: + description: "namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} restart." + summary: Pod restart alert + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: deq6i5dvnlkw0e + title: pod cpu usage over 80% + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: (sum(rate(container_cpu_usage_seconds_total{image!=""}[2m])) by (pod,namespace)/sum(kube_pod_container_resource_limits{resource="cpu"}) by (pod,namespace) * 100) > 80 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: "namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} 's cpu usage over {{$value }}%." + summary: CPU Usage of Pod limit on has exceeded 80%. + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: beq6l7fts9nnke + title: pod memory usage over 80% + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: (sum(container_memory_working_set_bytes{pod!="",container!=""}) by (pod, namespace)) / (sum(container_spec_memory_limit_bytes{pod!=""}) by (pod, namespace)) > 0.8 and (sum(container_memory_working_set_bytes{pod!="",container!=""}) by (pod, namespace)) / (sum (container_spec_memory_limit_bytes{pod!=""}) by (pod, namespace)) < 2 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + keepFiringFor: 2m + annotations: + description: "namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} 's memory usage over {{ $value }}%." + summary: Memory Usage of Pod limit on has exceeded 80%. + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: eeq6lm6biyfb4e + title: pod start fail + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: kube_pod_status_phase{pod!~".*job.*", phase=~"Failed|Unknown"} == 1 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: "namespace: {{ $labels.namespace }}, pod: {{ $labels.pod }} start fail." + summary: Pod start fail + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: feq6m6yr27togc + title: job exec error + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: kube_job_status_failed{job="drycc-kube-state-metrics"} == 1 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: "namespace: {{ $labels.namespace }}, job: {{ $labels.job_name }} exec {{ $labels.reason }}." + summary: Job exec error + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: beqhp6i29jx8gb + title: pvc usage over 80% + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: kubelet_volume_stats_used_bytes{job="kubernetes-nodes"} / kubelet_volume_stats_capacity_bytes{job="kubernetes-nodes"} > 0.8 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: pvc capacity {{$labels.namespace}} {{$labels.persistentvolumeclaim}} on {{$labels.topology_topolvm_io_node}} usage over {{$value }}%. + summary: Pvc usage over 80% + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: deqhpblphb7k0c + title: mysql down + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: mysql_up != 1 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: MySQL {{$labels.namespace}} {{$labels.pod}} on {{$labels.node}} is not up. + summary: MySQL not up. + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: feqhpe2zz56o0d + title: mysql group replcation down + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: mysql_perf_schema_replication_group_member_info{member_state!="ONLINE"} == 1 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: MySQL group replication {{$labels.namespace}} {{$labels.pod}} on {{$labels.node}} is not up. + summary: MySQL group replication not up. + isPaused: false + notification_settings: + receiver: grafana-default-email + - orgId: 1 + name: high + folder: drycc + interval: 1m + rules: + - uid: beq6ljd99ukg0c + title: node down + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: up{job="drycc-node-exporter"} == 0 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 1m + annotations: + description: "node: {{ $labels.instance }} down." + summary: Node down + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: eeq6lu5wz3eo0c + title: cluster node over pressure + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: kube_node_status_condition{condition=~"OutOfDisk|MemoryPressure|DiskPressure",status!="false"} == 1 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 1m + annotations: + description: "node: {{ $labels.node }} has {{ $labels.condition }}." + summary: Cluster node over pressure + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: deq6lwic91slce + title: cluster node status error + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: kube_node_status_condition{condition="Ready",status!="true"} == 1 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 1m + annotations: + description: "node: {{ $labels.node }} status error." + summary: Cluster node status error + isPaused: false + notification_settings: + receiver: grafana-default-email + - orgId: 1 + name: middle + folder: drycc + interval: 2m + rules: + - uid: beq6ptjdxwmpsc + title: node cpu usage over 80% + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: (1 - avg by (instance) (irate(node_cpu_seconds_total{job="drycc-node-exporter",mode="idle"}[5m]))) * 100 > 80 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: "instance: {{ $labels.instance }} cpu usage over {{ $value }}%" + summary: Node cpu high over 80 + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: ceqhohsqqyr5sb + title: node memory usage over 80% + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: (1-(node_memory_MemAvailable_bytes{job="drycc-node-exporter"} / (node_memory_MemTotal_bytes{job="drycc-node-exporter"})))* 100 > 80 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: "instance: {{ $labels.instance }} memory usage over {{ $value }}%." + summary: Node memory over 80 + isPaused: false + notification_settings: + receiver: grafana-default-email + - uid: aeqhp0wbcsb9cf + title: node systen disk usage over 80% + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: (1 - node_filesystem_avail_bytes{job="drycc-node-exporter",fstype=~"ext4|xfs"} / node_filesystem_size_bytes{job="drycc-node-exporter",fstype=~"ext4|xfs"}) * 100 > 80 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + for: 2m + annotations: + description: "instance: {{ $labels.instance }} disk usage over {{ $value }}%." + summary: Node systen disk usage over 80 + isPaused: false + notification_settings: + receiver: grafana-default-email + - orgId: 1 + name: slow + folder: drycc + interval: 1d + rules: + - uid: feq6lfrsmsum8a + title: node certificate expiration + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_on_drycc + model: + editorMode: code + expr: ((x509_cert_not_after - time()) / 86400) < 15 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: OK + execErrState: KeepLast + annotations: + description: "node: {{ $labels.node}} cert: {{ $labels.filepath }} will expire in {{ $labels.value }} days." + summary: Node certificate expiration + isPaused: false + notification_settings: + receiver: grafana-default-email diff --git a/rootfs/usr/share/grafana/provisioning/dashboards/drycc.yaml b/rootfs/usr/share/grafana/provisioning/dashboards/drycc.yaml index a047e87..aa68d28 100644 --- a/rootfs/usr/share/grafana/provisioning/dashboards/drycc.yaml +++ b/rootfs/usr/share/grafana/provisioning/dashboards/drycc.yaml @@ -2,8 +2,9 @@ apiVersion: 1 providers: - name: 'default' orgId: 1 - folder: 'drycc' - folderUid: 'drycc' type: file options: path: /usr/share/grafana/dashboards + folder: 'drycc' + allowUiUpdates: false + disableDeletion: true From 798d0bbaead9862e915bbd6a5b3e5c925f9c0eda Mon Sep 17 00:00:00 2001 From: lijianguo Date: Fri, 25 Jul 2025 15:40:40 +0800 Subject: [PATCH 38/50] chore(dashboards): update container network (#42) --- .../grafana/dashboards/kubernetes_deployment_metrics.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rootfs/usr/share/grafana/dashboards/kubernetes_deployment_metrics.json b/rootfs/usr/share/grafana/dashboards/kubernetes_deployment_metrics.json index cc9be62..a9062b7 100644 --- a/rootfs/usr/share/grafana/dashboards/kubernetes_deployment_metrics.json +++ b/rootfs/usr/share/grafana/dashboards/kubernetes_deployment_metrics.json @@ -1133,7 +1133,7 @@ { "datasource": null, "editorMode": "code", - "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",id!=\"\",pod=~\"^$Statefulset.*$\",namespace=\"${Namespace}\"}[2m])) by (pod)", + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",id!=\"\",pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"${Namespace}\"}[2m])) by (pod)", "interval": "10s", "intervalFactor": 1, "legendFormat": "-> {{ pod }}", @@ -1145,7 +1145,7 @@ { "datasource": null, "editorMode": "code", - "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",id!=\"\",pod=~\"^$statefulset.*$\",namespace=\"${namespace}\"}[2m])) by (pod)", + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",id!=\"\",pod=~\"^$Deployment.*$\",pod=~\"^$Statefulset.*$\",namespace=\"${Namespace}\"}[2m])) by (pod)", "interval": "10s", "intervalFactor": 1, "legendFormat": "<- {{ pod }}", From 4740c9a800cb5765e1c6bb6b8dc36f912df05338 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 25 Jul 2025 17:07:57 +0800 Subject: [PATCH 39/50] chore(grafana): change deployment to statefulset --- .../grafana/templates/grafana-configmap.yaml | 4 +- charts/grafana/templates/grafana-pvc.yaml | 24 ------------ charts/grafana/templates/grafana-service.yaml | 37 +++++++++++++++++++ ...ployment.yaml => grafana-statefulset.yaml} | 30 +++++++++------ charts/grafana/templates/grafana-svc.yaml | 21 ----------- charts/grafana/values.yaml | 2 +- rootfs/Dockerfile | 8 ++-- .../usr/share/grafana/oauth2/hook/__init__.py | 4 +- .../usr/share/grafana/oauth2/hook/grafana.py | 15 +++++++- 9 files changed, 79 insertions(+), 66 deletions(-) delete mode 100644 charts/grafana/templates/grafana-pvc.yaml create mode 100644 charts/grafana/templates/grafana-service.yaml rename charts/grafana/templates/{grafana-deployment.yaml => grafana-statefulset.yaml} (93%) delete mode 100644 charts/grafana/templates/grafana-svc.yaml diff --git a/charts/grafana/templates/grafana-configmap.yaml b/charts/grafana/templates/grafana-configmap.yaml index f741ae9..0c90538 100644 --- a/charts/grafana/templates/grafana-configmap.yaml +++ b/charts/grafana/templates/grafana-configmap.yaml @@ -37,9 +37,9 @@ data: app_mode = production [paths] - data = /var/lib/grafana + data = /opt/drycc/grafana/data home = /usr/share/grafana - logs = /var/log/grafana + logs = /opt/drycc/grafana/data/logs plugins = /usr/share/grafana/plugins provisioning = /usr/share/grafana/provisioning diff --git a/charts/grafana/templates/grafana-pvc.yaml b/charts/grafana/templates/grafana-pvc.yaml deleted file mode 100644 index 2032faa..0000000 --- a/charts/grafana/templates/grafana-pvc.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{{- if .Values.persistence.enabled }} -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: drycc-grafana - labels: - heritage: drycc -spec: - accessModes: - - {{ .Values.persistence.accessMode | quote }} - resources: - requests: - storage: {{ .Values.persistence.size | quote }} -{{- if .Values.persistence.storageClass }} -{{- if (eq "-" .Values.persistence.storageClass) }} - storageClassName: "" -{{- else }} - storageClassName: "{{ .Values.persistence.storageClass }}" -{{- end }} -{{- end }} -{{- if .Values.persistence.volumeName }} - volumeName: "{{ .Values.persistence.volumeName }}" -{{- end }} -{{- end }} diff --git a/charts/grafana/templates/grafana-service.yaml b/charts/grafana/templates/grafana-service.yaml new file mode 100644 index 0000000..d2e8494 --- /dev/null +++ b/charts/grafana/templates/grafana-service.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Service +metadata: + name: drycc-grafana + annotations: + {{- with .Values.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + heritage: drycc + app: drycc-grafana +spec: + type: ClusterIP + publishNotReadyAddresses: true + ports: + - name: grafana-tcp + port: 3000 + targetPort: grafana + protocol: TCP + - name: alerting-tcp + port: 9094 + targetPort: alerting + protocol: TCP + - name: alerting-udp + port: 9094 + targetPort: alerting + protocol: UDP + - name: oauth2-tcp + port: 4000 + targetPort: oauth2 + protocol: TCP + - name: proxy-tcp + port: 80 + targetPort: proxy + protocol: TCP + selector: + app: drycc-grafana diff --git a/charts/grafana/templates/grafana-deployment.yaml b/charts/grafana/templates/grafana-statefulset.yaml similarity index 93% rename from charts/grafana/templates/grafana-deployment.yaml rename to charts/grafana/templates/grafana-statefulset.yaml index 7e653c5..2895160 100644 --- a/charts/grafana/templates/grafana-deployment.yaml +++ b/charts/grafana/templates/grafana-statefulset.yaml @@ -1,5 +1,5 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: drycc-grafana labels: @@ -8,11 +8,9 @@ metadata: component.drycc.cc/version: {{ .Values.imageTag }} spec: replicas: 1 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 + updateStrategy: type: RollingUpdate + podManagementPolicy: Parallel selector: matchLabels: app: drycc-grafana @@ -194,7 +192,7 @@ spec: {{- toYaml . | nindent 10 }} {{- end }} ports: - - containerPort: 5000 + - containerPort: 80 name: proxy {{- include "grafana.envs" . | indent 8 }} {{- if not .Values.diagnosticMode.enabled }} @@ -226,16 +224,26 @@ spec: successThreshold: 1 {{- end }} volumeMounts: + {{- if .Values.persistence.enabled }} + - name: grafana-data + mountPath: /opt/drycc/grafana/data + {{- end }} - name: grafana-config subPath: Caddyfile readOnly: true mountPath: /usr/share/grafana/Caddyfile volumes: - {{- if .Values.persistence.enabled }} - - name: grafana-data - persistentVolumeClaim: - claimName: drycc-grafana - {{- end }} - name: grafana-config configMap: name: grafana-config + {{- if .Values.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: grafana-data + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: {{ .Values.persistence.storageClass }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- end }} diff --git a/charts/grafana/templates/grafana-svc.yaml b/charts/grafana/templates/grafana-svc.yaml deleted file mode 100644 index 01bb4a2..0000000 --- a/charts/grafana/templates/grafana-svc.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: drycc-grafana - annotations: - {{- with .Values.service.annotations }} - {{- toYaml . | nindent 4 }} - {{- end }} - labels: - heritage: drycc - app: drycc-grafana -spec: - ports: - - port: 80 - name: grafana - targetPort: proxy - - port: 9094 - name: alerting - targetPort: alerting - selector: - app: drycc-grafana diff --git a/charts/grafana/values.yaml b/charts/grafana/values.yaml index bb8e9d3..08d7b1e 100644 --- a/charts/grafana/values.yaml +++ b/charts/grafana/values.yaml @@ -53,8 +53,8 @@ service: persistence: enabled: false - accessMode: ReadWriteOnce size: 5Gi + storageClass: "" # Any custom grafana environment variables # can be specified as key-value pairs under environment # this is usually a non required setting. diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index cbac2e3..16a20d4 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -25,11 +25,11 @@ RUN install-packages unzip fontconfig \ && chown -R ${DRYCC_UID}:${DRYCC_GID} ${DRYCC_HOME_DIR} \ # set env && echo "source ${DRYCC_HOME_DIR}/venv/bin/activate" >> /opt/drycc/python/profile.d/python.sh \ - # grafana plugin + # grafana plugins && mkdir ${DRYCC_HOME_DIR}/plugins \ - && curl -sSL https://github.com/quickwit-oss/quickwit-datasource/releases/download/v0.4.6/quickwit-quickwit-datasource-0.4.6.zip -o quickwit.zip \ - && unzip quickwit.zip -d ${DRYCC_HOME_DIR}/plugins \ - && rm -rf quickwit.zip \ + && grafana cli --pluginsDir=${DRYCC_HOME_DIR}/plugins --homepath=/opt/drycc/grafana plugins install grafana-llm-app \ + && grafana cli --pluginsDir=${DRYCC_HOME_DIR}/plugins --homepath=/opt/drycc/grafana plugins install quickwit-quickwit-datasource \ + # clean && rm -rf \ /usr/share/doc \ /usr/share/man \ diff --git a/rootfs/usr/share/grafana/oauth2/hook/__init__.py b/rootfs/usr/share/grafana/oauth2/hook/__init__.py index 89233ba..5e15b49 100644 --- a/rootfs/usr/share/grafana/oauth2/hook/__init__.py +++ b/rootfs/usr/share/grafana/oauth2/hook/__init__.py @@ -1,9 +1,9 @@ from .grafana import ( - init_org, sync_user, sync_role, sync_datasources, sync_folder, sync_dashboards, sync_alerting + init_org, sync_user, sync_role, sync_default, sync_datasources, sync_dashboards, sync_alerting ) startup_hooks = [init_org] login_hooks = [ - sync_user, sync_role, sync_datasources, sync_folder, sync_dashboards, sync_alerting + sync_user, sync_role, sync_default, sync_datasources, sync_dashboards, sync_alerting ] destroy_hooks = [] diff --git a/rootfs/usr/share/grafana/oauth2/hook/grafana.py b/rootfs/usr/share/grafana/oauth2/hook/grafana.py index e4943ec..24f9d23 100644 --- a/rootfs/usr/share/grafana/oauth2/hook/grafana.py +++ b/rootfs/usr/share/grafana/oauth2/hook/grafana.py @@ -82,13 +82,26 @@ async def sync_role(context: dict, token: dict, userinfo: dict): await init_org(org_id=context["org_id"], name=userinfo["preferred_username"]) -async def sync_folder(context: dict, token: dict, userinfo: dict): +async def sync_default(context: dict, token: dict, userinfo: dict): async with httpx.AsyncClient() as client: await client.post( api_url("/api/folders"), headers=api_headers(context, userinfo), json={"uid": "drycc", "title": "drycc"}, ) + await client.post( + api_url("/api/v1/provisioning/contact-points"), + headers=api_headers(context, userinfo), + json={ + "uid": "grafana-default-email", + "name": "grafana-default-email", + "type": "email", + "settings": { + "addresses": f"{userinfo["email"]}" + }, + "disableResolveMessage": False + }, + ) async def sync_alerting(context: dict, token: dict, userinfo: dict): From 83409db51e6ff8c7407511d4fd4e27fc7ae681d5 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Sun, 27 Jul 2025 19:21:34 +0800 Subject: [PATCH 40/50] chore(hook): remove create contact_points --- .../grafana/templates/grafana-configmap.yaml | 2 +- .../usr/share/grafana/oauth2/hook/grafana.py | 46 +++++++++++++------ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/charts/grafana/templates/grafana-configmap.yaml b/charts/grafana/templates/grafana-configmap.yaml index 0c90538..a681f9f 100644 --- a/charts/grafana/templates/grafana-configmap.yaml +++ b/charts/grafana/templates/grafana-configmap.yaml @@ -14,7 +14,7 @@ data: output stdout } } - :5000 { + :80 { handle /oauth2/* { reverse_proxy 127.0.0.1:4000 } diff --git a/rootfs/usr/share/grafana/oauth2/hook/grafana.py b/rootfs/usr/share/grafana/oauth2/hook/grafana.py index 24f9d23..fc2a2fd 100644 --- a/rootfs/usr/share/grafana/oauth2/hook/grafana.py +++ b/rootfs/usr/share/grafana/oauth2/hook/grafana.py @@ -1,4 +1,5 @@ import os +import time import json import httpx from string import Template @@ -83,25 +84,44 @@ async def sync_role(context: dict, token: dict, userinfo: dict): async def sync_default(context: dict, token: dict, userinfo: dict): + alertmanager_default_config = json.dumps({ + "alertmanager_config": { + "route": { + "receiver": "grafana-default-email", + "group_by": ["grafana_folder", "alertname"] + }, + "receivers": [{ + "name": "grafana-default-email", + "grafana_managed_receiver_configs": [{ + "uid": "", + "name": "email receiver", + "type": "email", + "settings": { + "addresses": f"{userinfo["email"]}" + } + }] + }] + } + }) async with httpx.AsyncClient() as client: await client.post( api_url("/api/folders"), headers=api_headers(context, userinfo), json={"uid": "drycc", "title": "drycc"}, ) - await client.post( - api_url("/api/v1/provisioning/contact-points"), - headers=api_headers(context, userinfo), - json={ - "uid": "grafana-default-email", - "name": "grafana-default-email", - "type": "email", - "settings": { - "addresses": f"{userinfo["email"]}" - }, - "disableResolveMessage": False - }, - ) + sql_tpl = f""" + INSERT INTO alert_configuration ( + alertmanager_configuration, configuration_version, created_at, "default", org_id + ) + VALUES ( + '{alertmanager_default_config}', 'v1', {int(time.time())}, TRUE, {context["org_id"]} + ) + ON CONFLICT (org_id) DO NOTHING; + """ + async with await AsyncConnection.connect(os.environ.get("GF_DATABASE_URL")) as conn: + async with conn.cursor() as cursor: + await cursor.execute(sql_tpl) + await conn.commit() async def sync_alerting(context: dict, token: dict, userinfo: dict): From 46de839d7a25041cccf829a852488a44daf67fcc Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 2 Sep 2025 11:37:10 +0800 Subject: [PATCH 41/50] feat(charts): add grafana migration init container --- .../templates/grafana-statefulset.yaml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/charts/grafana/templates/grafana-statefulset.yaml b/charts/grafana/templates/grafana-statefulset.yaml index 2895160..8a48ac2 100644 --- a/charts/grafana/templates/grafana-statefulset.yaml +++ b/charts/grafana/templates/grafana-statefulset.yaml @@ -37,6 +37,39 @@ spec: - -u - $(GF_DATABASE_URL),http://drycc-controller-api {{- include "grafana.envs" . | indent 8 }} + - name: drycc-grafana-migrate + image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/grafana:{{.Values.imageTag}} + imagePullPolicy: {{.Values.imagePullPolicy}} + args: + - /usr/bin/env + - bash + - -ec + - | + set -m + echo "Configure Grafana live HA engine" + DRYCC_VALKEY_JSON=$(echo "$DRYCC_VALKEY_URL" | jq -R 'capture("^(?redis[s]?)://(?[^:]+)?(?::(?[^@]+))?@(?
[^/]+)/?(?[0-9]*)$")') + GF_LIVE_HA_ENGINE_ADDRESS=$(echo "${DRYCC_VALKEY_JSON}" |jq -r '.address') + GF_LIVE_HA_ENGINE_PASSWORD=$(echo "${DRYCC_VALKEY_JSON}" |jq -r '.password') + export GF_LIVE_HA_ENGINE_ADDRESS GF_LIVE_HA_ENGINE_PASSWORD + grafana server --config /usr/share/grafana/grafana.ini --homepath /opt/drycc/grafana & + GRAFANA_PID=$! + echo "Waiting for Grafana to come up..." + until curl -q --fail --output /dev/null --silent "http://localhost:3000/api/health"; do + printf "." + sleep 2 + done + kill $GRAFANA_PID 2>/dev/null + wait $GRAFANA_PID 2>/dev/null + {{- include "grafana.envs" . | indent 8 }} + volumeMounts: + {{- if .Values.persistence.enabled }} + - name: grafana-data + mountPath: /var/lib/grafana + {{- end }} + - name: grafana-config + subPath: grafana.ini + readOnly: true + mountPath: /usr/share/grafana/grafana.ini containers: - name: drycc-grafana image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/grafana:{{.Values.imageTag}} From 607d6a7e387c6e4e143501c44147335bfb002d6d Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Sat, 13 Sep 2025 23:37:25 +0800 Subject: [PATCH 42/50] chore(grafana): bump new version --- rootfs/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 16a20d4..76e694d 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -4,9 +4,9 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/usr/share/grafana \ - GRAFANA_VERSION="12.1.0" \ + GRAFANA_VERSION="12.1.1" \ JQ_VERSION="1.7.1" \ - CADDY_VERSION="2.9.1" \ + CADDY_VERSION="2.10.2" \ PYTHON_VERSION="3.13" RUN groupadd drycc --gid ${DRYCC_GID} \ From 5f8101639e99ccdca18b9fb596ec26393a779c45 Mon Sep 17 00:00:00 2001 From: Eamon Date: Tue, 30 Sep 2025 15:20:25 +0800 Subject: [PATCH 43/50] chore(grafana): modify oauth2 dashboards (#43) Co-authored-by: zhangeamon --- .../oauth2/dashboards/drycc_applicaion.json | 8 +- .../oauth2/dashboards/drycc_gateway.json | 581 ++++-------------- 2 files changed, 131 insertions(+), 458 deletions(-) diff --git a/rootfs/usr/share/grafana/oauth2/dashboards/drycc_applicaion.json b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_applicaion.json index bee7b27..0abe8c7 100644 --- a/rootfs/usr/share/grafana/oauth2/dashboards/drycc_applicaion.json +++ b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_applicaion.json @@ -581,7 +581,7 @@ { "datasource": { "type": "prometheus", - "uid": "PF6837BEDE254B550" + "uid": "prometheus_on_drycc" }, "editorMode": "code", "expr": "", @@ -764,7 +764,7 @@ { "datasource": { "type": "prometheus", - "uid": "PF6837BEDE254B550" + "uid": "prometheus_on_drycc" }, "editorMode": "code", "expr": "sum(rate(container_network_receive_bytes_total{namespace=~\"$app\"}[5m])) by (pod)", @@ -871,7 +871,7 @@ { "datasource": { "type": "prometheus", - "uid": "PF6837BEDE254B550" + "uid": "prometheus_on_drycc" }, "editorMode": "code", "expr": "sum(rate(container_network_transmit_bytes_total{namespace=~\"$app\"}[5m])) by (pod)", @@ -992,7 +992,7 @@ { "datasource": { "type": "prometheus", - "uid": "PF6837BEDE254B550" + "uid": "prometheus_on_drycc" }, "editorMode": "code", "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$app\",container!=\"\"}[1m])) by(pod)", diff --git a/rootfs/usr/share/grafana/oauth2/dashboards/drycc_gateway.json b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_gateway.json index 82152e0..78f2d48 100644 --- a/rootfs/usr/share/grafana/oauth2/dashboards/drycc_gateway.json +++ b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_gateway.json @@ -12,10 +12,10 @@ ] }, "description": "General Drycc Gateway metrics.", - "editable": false, + "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 79, + "id": 20, "links": [], "panels": [ { @@ -57,7 +57,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -71,7 +72,7 @@ }, "gridPos": { "h": 4, - "w": 8, + "w": 6, "x": 0, "y": 1 }, @@ -84,7 +85,9 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -92,11 +95,11 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "12.0.2", + "pluginVersion": "12.1.1", "targets": [ { "editorMode": "code", - "expr": "round(sum(irate(istio_requests_total{destination_service=~\"$service\"}[$__rate_interval])), 0.001)", + "expr": "round(sum(irate(istio_requests_total{destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])), 0.001)", "format": "time_series", "intervalFactor": 1, "range": true, @@ -107,76 +110,6 @@ "title": "Client Request Volume", "type": "stat" }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus_on_drycc" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "rgb(31, 120, 193)", - "mode": "fixed" - }, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-red" - }, - { - "color": "dark-yellow", - "value": 0.95 - }, - { - "color": "dark-green", - "value": 0.99 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 8, - "y": 1 - }, - "id": 14, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(irate(istio_requests_total{reporter=~\"$qrep\",destination_service=~\"$service\",response_code!~\"5.*\"}[$__rate_interval])) / (sum(irate(istio_requests_total{reporter=~\"$qrep\",destination_service=~\"$service\"}[$__rate_interval])) or on () vector(1))", - "format": "time_series", - "intervalFactor": 1, - "range": true, - "refId": "A" - } - ], - "title": "Client Success Rate (non-5xx responses)", - "type": "stat" - }, { "datasource": { "type": "prometheus", @@ -225,7 +158,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -239,8 +173,8 @@ }, "gridPos": { "h": 4, - "w": 8, - "x": 16, + "w": 6, + "x": 6, "y": 1 }, "id": 87, @@ -257,14 +191,16 @@ "sort": "none" } }, - "pluginVersion": "12.0.2", + "pluginVersion": "12.1.1", "targets": [ { - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\",destination_service=~\"$service\"}[$__rate_interval])) by (le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\",destination_service=~\"$service\"}[$__rate_interval])) by (le))", + "editorMode": "code", + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\",destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\",destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (le))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "P50", + "range": true, "refId": "A" }, { @@ -304,7 +240,8 @@ "mode": "absolute", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)" + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", @@ -322,9 +259,9 @@ }, "gridPos": { "h": 4, - "w": 8, - "x": 0, - "y": 5 + "w": 6, + "x": 12, + "y": 1 }, "id": 98, "maxDataPoints": 100, @@ -335,7 +272,9 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -343,11 +282,11 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "12.0.2", + "pluginVersion": "12.1.1", "targets": [ { "editorMode": "code", - "expr": "sum(irate(istio_requests_total{reporter=\"destination\",destination_service=~\"$service\",response_code!~\"5.*\"}[$__rate_interval])) / (sum(irate(istio_requests_total{reporter=\"destination\",destination_service=~\"$service\"}[$__rate_interval])) or on () vector(1))", + "expr": "sum(irate(istio_requests_total{reporter=\"destination\",destination_service=~\"$service\",response_code!~\"5.*\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) / (sum(irate(istio_requests_total{reporter=\"destination\",destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) or on () vector(1))", "format": "time_series", "intervalFactor": 1, "range": true, @@ -368,115 +307,36 @@ "fixedColor": "rgb(31, 120, 193)", "mode": "fixed" }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], + "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 8, - "y": 5 - }, - "id": 100, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["mean"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.2", - "targets": [ - { - "expr": "sum(irate(istio_tcp_sent_bytes_total{reporter=~\"$qrep\", destination_service=~\"$service\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "TCP Sent Bytes", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus_on_drycc" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "rgb(31, 120, 193)", - "mode": "fixed" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } + "color": "dark-red", + "value": 0 }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ { - "color": "green" + "color": "dark-yellow", + "value": 0.95 }, { - "color": "red", - "value": 80 + "color": "dark-green", + "value": 0.99 } ] }, - "unit": "Bps" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 4, - "w": 8, - "x": 16, - "y": 5 + "w": 6, + "x": 18, + "y": 1 }, - "id": 84, + "id": 14, "maxDataPoints": 100, "options": { "colorMode": "none", @@ -485,7 +345,9 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["mean"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -493,18 +355,18 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "12.0.2", + "pluginVersion": "12.1.1", "targets": [ { - "expr": "sum(irate(istio_tcp_received_bytes_total{reporter=~\"$qrep\", destination_service=~\"$service\"}[$__rate_interval]))", + "editorMode": "code", + "expr": "sum(irate(istio_requests_total{reporter=~\"$qrep\",destination_service=~\"$service\",response_code!~\"5.*\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) / (sum(irate(istio_requests_total{reporter=~\"$qrep\",destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) or on () vector(1))", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "", + "range": true, "refId": "A" } ], - "title": "TCP Received Bytes", + "title": "Client Success Rate (non-5xx responses)", "type": "stat" }, { @@ -513,7 +375,7 @@ "h": 1, "w": 24, "x": 0, - "y": 9 + "y": 5 }, "id": 104, "panels": [], @@ -569,7 +431,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -606,7 +469,7 @@ "h": 6, "w": 12, "x": 0, - "y": 10 + "y": 6 }, "id": 25, "options": { @@ -622,11 +485,11 @@ "sort": "none" } }, - "pluginVersion": "12.0.2", + "pluginVersion": "12.1.1", "targets": [ { "editorMode": "code", - "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\",destination_service=~\"$service\",reporter=~\"$qrep\"}[$__rate_interval])) by (source_workload, source_workload_namespace,response_code), 0.001)", + "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\",destination_service=~\"$service\",reporter=~\"$qrep\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace,response_code), 0.001)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -637,7 +500,7 @@ }, { "editorMode": "code", - "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", reporter=~\"$qrep\"}[$__rate_interval])) by (source_workload, source_workload_namespace, response_code), 0.001)", + "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", reporter=~\"$qrep\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, response_code), 0.001)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -700,7 +563,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -737,7 +601,7 @@ "h": 6, "w": 12, "x": 12, - "y": 10 + "y": 6 }, "id": 26, "options": { @@ -753,11 +617,11 @@ "sort": "none" } }, - "pluginVersion": "12.0.2", + "pluginVersion": "12.1.1", "targets": [ { "editorMode": "code", - "expr": "sum(irate(istio_requests_total{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\"}[$__rate_interval])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace)", + "expr": "sum(irate(istio_requests_total{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -768,7 +632,7 @@ }, { "editorMode": "code", - "expr": "sum(irate(istio_requests_total{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\", }[$__rate_interval])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace)", + "expr": "sum(irate(istio_requests_total{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\",namespace=\"$app\",pod=\"$instance\" }[$__rate_interval])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -830,7 +694,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -867,7 +732,7 @@ "h": 6, "w": 8, "x": 0, - "y": 16 + "y": 12 }, "id": 27, "options": { @@ -883,11 +748,11 @@ "sort": "none" } }, - "pluginVersion": "12.0.2", + "pluginVersion": "12.1.1", "targets": [ { "editorMode": "code", - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -898,7 +763,7 @@ }, { "editorMode": "code", - "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -909,7 +774,7 @@ }, { "editorMode": "code", - "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -920,7 +785,7 @@ }, { "editorMode": "code", - "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -931,7 +796,7 @@ }, { "editorMode": "code", - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -942,7 +807,7 @@ }, { "editorMode": "code", - "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -953,7 +818,7 @@ }, { "editorMode": "code", - "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -964,7 +829,7 @@ }, { "editorMode": "code", - "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1026,7 +891,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -1063,7 +929,7 @@ "h": 6, "w": 8, "x": 8, - "y": 16 + "y": 12 }, "id": 28, "options": { @@ -1079,11 +945,11 @@ "sort": "none" } }, - "pluginVersion": "12.0.2", + "pluginVersion": "12.1.1", "targets": [ { "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1094,7 +960,7 @@ }, { "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1105,7 +971,7 @@ }, { "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1116,7 +982,7 @@ }, { "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1127,7 +993,7 @@ }, { "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1138,7 +1004,7 @@ }, { "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1149,7 +1015,7 @@ }, { "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1160,7 +1026,7 @@ }, { "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1222,7 +1088,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -1259,7 +1126,7 @@ "h": 6, "w": 8, "x": 16, - "y": 16 + "y": 12 }, "id": 68, "options": { @@ -1275,11 +1142,11 @@ "sort": "none" } }, - "pluginVersion": "12.0.2", + "pluginVersion": "12.1.1", "targets": [ { "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1290,7 +1157,7 @@ }, { "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1301,7 +1168,7 @@ }, { "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1312,7 +1179,7 @@ }, { "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1323,7 +1190,7 @@ }, { "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1334,7 +1201,7 @@ }, { "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1345,7 +1212,7 @@ }, { "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1356,7 +1223,7 @@ }, { "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",namespace=\"$app\",pod=\"$instance\"}[$__rate_interval])) by (source_workload, source_workload_namespace, le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1368,231 +1235,21 @@ ], "title": "Response Size By Source", "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus_on_drycc" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 22 - }, - "id": 80, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.2", - "targets": [ - { - "editorMode": "code", - "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=~\"$qrep\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace), 0.001)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", - "range": true, - "refId": "A", - "step": 2 - }, - { - "editorMode": "code", - "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=~\"$qrep\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace), 0.001)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", - "range": true, - "refId": "B", - "step": 2 - } - ], - "title": "Bytes Received from Incoming TCP Connection", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus_on_drycc" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 22 - }, - "id": 82, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.0.2", - "targets": [ - { - "editorMode": "code", - "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy=\"mutual_tls\", reporter=~\"$qrep\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", - "range": true, - "refId": "A", - "step": 2 - }, - { - "editorMode": "code", - "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy!=\"mutual_tls\", reporter=~\"$qrep\", destination_service=~\"$service\"}[$__rate_interval])) by (source_workload, source_workload_namespace), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", - "range": true, - "refId": "B", - "step": 2 - } - ], - "title": "Bytes Sent to Incoming TCP Connection", - "type": "timeseries" } ], "preload": false, - "refresh": "1m", + "refresh": "60s", "schemaVersion": 41, - "tags": ["drycc"], + "tags": [ + "drycc" + ], "templating": { "list": [ { + "current": { + "text": "demo3", + "value": "demo3" + }, "definition": "label_values(istio_requests_total,namespace)", "description": "namespace ", "label": "app", @@ -1609,6 +1266,10 @@ }, { "allowCustomValue": false, + "current": { + "text": "demo3-istio-58647c8789-ztcql", + "value": "demo3-istio-58647c8789-ztcql" + }, "definition": "label_values(istio_requests_total{namespace=\"$app\"},pod)", "description": "pod ", "label": "instance", @@ -1624,8 +1285,13 @@ "type": "query" }, { - "allValue": "", "allowCustomValue": false, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, "definition": "query_result(sum(istio_requests_total{namespace=\"$app\",pod=\"$instance\"}) by (destination_service) or sum(istio_tcp_sent_bytes_total{namespace=\"$app\",pod=\"$instance\"}) by (destination_service))", "includeAll": true, "label": "Service", @@ -1664,14 +1330,21 @@ ] }, "time": { - "from": "now-15m", + "from": "now-6h", "to": "now" }, "timepicker": { - "refresh_intervals": ["5m", "15m", "30m", "1h", "2h", "1d"] + "refresh_intervals": [ + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] }, "timezone": "", "title": "Drycc Gateway", "uid": "bb2f5242-9429-4f60-b520-20b7c70a0a9a", - "version": 12 -} + "version": 4 +} \ No newline at end of file From ecf1a32d45c1568e5371389501deb368aa555a17 Mon Sep 17 00:00:00 2001 From: Eamon Date: Tue, 14 Oct 2025 10:42:39 +0800 Subject: [PATCH 44/50] fix(dashboards)dyrcc volume dashboard (#44) * chore(grafana): modify oauth2 dashboards * fix(dashboards)dyrcc volume dashboard --------- Co-authored-by: zhangeamon --- .../grafana/oauth2/dashboards/drycc_volume.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rootfs/usr/share/grafana/oauth2/dashboards/drycc_volume.json b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_volume.json index d044890..e12b014 100644 --- a/rootfs/usr/share/grafana/oauth2/dashboards/drycc_volume.json +++ b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_volume.json @@ -103,7 +103,7 @@ "targets": [ { "editorMode": "code", - "expr": "(\n sum without(instance, node) (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "Used Space", @@ -112,7 +112,7 @@ }, { "editorMode": "code", - "expr": "sum without(instance, node) (kubelet_volume_stats_available_bytes{job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "expr": "sum without(instance, node) (kubelet_volume_stats_available_bytes{ namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "Free Space", @@ -191,7 +191,7 @@ "targets": [ { "editorMode": "code", - "expr": "(\n kubelet_volume_stats_capacity_bytes{job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{ job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "expr": "(\n kubelet_volume_stats_capacity_bytes{ namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{ namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{ namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -287,7 +287,7 @@ "targets": [ { "editorMode": "code", - "expr": "sum without(instance, node) (kubelet_volume_stats_inodes_used{ job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "expr": "sum without(instance, node) (kubelet_volume_stats_inodes_used{ namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "Used inodes", @@ -296,7 +296,7 @@ }, { "editorMode": "code", - "expr": "(\n sum without(instance, node) (kubelet_volume_stats_inodes{job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_inodes_used{job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_inodes{ namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_inodes_used{ namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", "format": "time_series", "intervalFactor": 1, "legendFormat": " Free inodes", @@ -375,7 +375,7 @@ "targets": [ { "editorMode": "code", - "expr": "kubelet_volume_stats_inodes_used{job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{job=\"kubernetes-apiservers\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "expr": "kubelet_volume_stats_inodes_used{ namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{ namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "", From 05e9432dfd4bab9633ccbae4a2b81b763c1a1d98 Mon Sep 17 00:00:00 2001 From: lijianguo Date: Fri, 24 Oct 2025 15:45:17 +0800 Subject: [PATCH 45/50] chore(dashboard): drycc applicaion memory show podname (#45) --- .../usr/share/grafana/oauth2/dashboards/drycc_applicaion.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rootfs/usr/share/grafana/oauth2/dashboards/drycc_applicaion.json b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_applicaion.json index 0abe8c7..2457a88 100644 --- a/rootfs/usr/share/grafana/oauth2/dashboards/drycc_applicaion.json +++ b/rootfs/usr/share/grafana/oauth2/dashboards/drycc_applicaion.json @@ -574,7 +574,7 @@ "expr": "(container_memory_working_set_bytes{namespace=~\"$app\",container!=\"\"}) \n", "hide": false, "interval": "", - "legendFormat": "Used Memory", + "legendFormat": "{{pod}}", "range": true, "refId": "A" }, From 2216c2b06ffe8117355ffd023fbb8df8d396cac4 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Thu, 6 Nov 2025 16:03:11 +0800 Subject: [PATCH 46/50] chore(grafana): change env to build arg --- rootfs/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 76e694d..ae60495 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -1,7 +1,7 @@ ARG CODENAME FROM registry.drycc.cc/drycc/base:${CODENAME} -ENV DRYCC_UID=1001 \ +ARG DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/usr/share/grafana \ GRAFANA_VERSION="12.1.1" \ From 70bd9a4172e6d8ac554fa508bd5d771fc09e8d1d Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Sat, 15 Nov 2025 20:55:19 +0800 Subject: [PATCH 47/50] chore(grafana): bump version to 12.2.1 --- rootfs/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index ae60495..2515407 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -4,10 +4,10 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ARG DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/usr/share/grafana \ - GRAFANA_VERSION="12.1.1" \ + GRAFANA_VERSION="12.2.1" \ JQ_VERSION="1.7.1" \ CADDY_VERSION="2.10.2" \ - PYTHON_VERSION="3.13" + PYTHON_VERSION="3.14" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} From 7df708a9eee7c78c62b47b622528c4d0ca44e756 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Tue, 9 Dec 2025 16:49:10 +0800 Subject: [PATCH 48/50] chore(grafana): switch redis db to 2 --- charts/grafana/templates/_helpers.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/grafana/templates/_helpers.tmpl b/charts/grafana/templates/_helpers.tmpl index a5d169b..b61bb01 100644 --- a/charts/grafana/templates/_helpers.tmpl +++ b/charts/grafana/templates/_helpers.tmpl @@ -19,7 +19,7 @@ env: name: valkey-creds key: password - name: DRYCC_VALKEY_URL - value: "redis://:$(DRYCC_VALKEY_PASSWORD)@drycc-valkey:16379/0" + value: "redis://:$(DRYCC_VALKEY_PASSWORD)@drycc-valkey:16379/2" {{- end }} - name: DRYCC_SERVICE_KEY valueFrom: From 29092f1d2a654c97384c5b5dd475a10cf6494586 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 10 Apr 2026 13:10:57 +0800 Subject: [PATCH 49/50] feat(workspace): migrate controller from owner-based auth to workspace model --- .../grafana/templates/grafana-configmap.yaml | 7 +- rootfs/Dockerfile | 4 +- .../oauth2/datasources/prometheus.json | 2 +- .../grafana/oauth2/datasources/quickwit.json | 2 +- .../usr/share/grafana/oauth2/hook/grafana.py | 509 +++++++++++++----- .../provisioning/alerting/contactpoints.yaml | 11 + 6 files changed, 393 insertions(+), 142 deletions(-) create mode 100644 rootfs/usr/share/grafana/provisioning/alerting/contactpoints.yaml diff --git a/charts/grafana/templates/grafana-configmap.yaml b/charts/grafana/templates/grafana-configmap.yaml index a681f9f..f1c6b84 100644 --- a/charts/grafana/templates/grafana-configmap.yaml +++ b/charts/grafana/templates/grafana-configmap.yaml @@ -63,7 +63,8 @@ data: [users] allow_sign_up = false allow_org_create = false - auto_assign_org = false + auto_assign_org = true + auto_assign_org_role = None login_hint = email or username [auth] @@ -102,12 +103,12 @@ data: [quota] enabled = true - org_user = 2 + org_user = -1 org_dashboard = 20 org_data_source = 5 org_api_key = 2 org_alert_rule = 20 - user_org = 2 + user_org = -1 alerting_rule_group_rules = 20 alerting_rule_evaluation_results = 20 diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 2515407..1824c4d 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -4,9 +4,9 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ARG DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/usr/share/grafana \ - GRAFANA_VERSION="12.2.1" \ + GRAFANA_VERSION="12.4.2" \ JQ_VERSION="1.7.1" \ - CADDY_VERSION="2.10.2" \ + CADDY_VERSION="2.11.2" \ PYTHON_VERSION="3.14" RUN groupadd drycc --gid ${DRYCC_GID} \ diff --git a/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json b/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json index 4eefc33..aed8d47 100644 --- a/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json +++ b/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json @@ -2,7 +2,7 @@ "name": "Prometheus on Drycc", "type": "prometheus", "uid": "prometheus_on_drycc", - "url": "${controller_url}/v2/prometheus/${username}", + "url": "${controller_url}/v2/prometheus/${workspace}", "access": "proxy", "isDefault": true, "basicAuth": false, diff --git a/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json b/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json index 8c1861b..1d1f012 100644 --- a/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json +++ b/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json @@ -2,7 +2,7 @@ "name": "Application Logs", "type": "quickwit-quickwit-datasource", "uid": "application_logs", - "url": "${controller_url}/v2/quickwit/${username}", + "url": "${controller_url}/v2/quickwit/${workspace}", "access": "proxy", "basicAuth": false, "jsonData": { diff --git a/rootfs/usr/share/grafana/oauth2/hook/grafana.py b/rootfs/usr/share/grafana/oauth2/hook/grafana.py index fc2a2fd..9d25a24 100644 --- a/rootfs/usr/share/grafana/oauth2/hook/grafana.py +++ b/rootfs/usr/share/grafana/oauth2/hook/grafana.py @@ -1,3 +1,4 @@ +import logging import os import time import json @@ -5,42 +6,31 @@ from string import Template from psycopg import AsyncConnection +logger = logging.getLogger(__name__) + DEFAULT_HEADERS = {"Content-Type": "application/json"} DRYCC_CONTROLLER_URL = os.environ.get('DRYCC_CONTROLLER_URL') DRYCC_GRAFANA_REFRESH = os.environ.get('DRYCC_GRAFANA_REFRESH', '60s') DRYCC_GRAFANA_DASHBOARD = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../") - -def api_url(url_path, is_admin=False): - if is_admin: - return "http://{}:{}@localhost:{}{}".format( - os.environ.get('GF_SECURITY_ADMIN_USER'), - os.environ.get('GF_SECURITY_ADMIN_PASSWORD'), - os.environ.get('GF_SERVER_HTTP_PORT', 3000), - url_path, - ) - return "http://localhost:{}{}".format(os.environ.get('GF_SERVER_HTTP_PORT', 3000), url_path) +# Drycc Workspace role to Grafana role mapping +DRYCC_WORKSPACE_ROLE_MAPPING = {"admin": "Editor", "member": "Editor", "viewer": "Viewer"} +# Grafana default main organization ID, renamed to "drycc" by init_org on startup +DRYCC_ORG_ID = 1 -def api_headers(context: dict, userinfo): - headers = {"Content-Type": "application/json"} - headers["Remote-User"] = userinfo["preferred_username"] - headers["Remote-Name"] = userinfo["preferred_username"] - headers["Remote-Email"] = userinfo["email"] - if "org_id" in context: - headers["X-Grafana-Org-Id"] = str(context["org_id"]) - return headers +# ── Public functions (exported via __init__.py) ────────────────────────── -async def init_org(org_id=1, name="drycc"): +async def init_org(org_id=DRYCC_ORG_ID, name="drycc"): headers = {"Content-Type": "application/json"} async with httpx.AsyncClient() as client: - resp = await client.get(api_url(f"/api/orgs/{org_id}", is_admin=True), headers=headers) + resp = await client.get(_api_url(f"/api/orgs/{org_id}", is_admin=True), headers=headers) org = resp.json() if org["name"] == name: return await client.put( - api_url(f"/api/orgs/{org_id}", is_admin=True), + _api_url(f"/api/orgs/{org_id}", is_admin=True), headers=headers, json={"name": name}, ) @@ -48,153 +38,401 @@ async def init_org(org_id=1, name="drycc"): async def sync_user(context: dict, token: dict, userinfo: dict): async with httpx.AsyncClient() as client: - resp = await client.get(api_url("/api/user"), headers=api_headers(context, userinfo)) + resp = await client.get(_api_url("/api/user"), headers=_api_headers(context, userinfo)) user = resp.json() context["user_id"] = user["id"] if userinfo["is_superuser"]: await client.put( - api_url(f"/api/admin/users/{context["user_id"]}/permissions", is_admin=True), - headers=api_headers(context, userinfo), + _api_url(f"/api/admin/users/{context['user_id']}/permissions", is_admin=True), + headers=_api_headers(context, userinfo), json={"isGrafanaAdmin": userinfo["is_superuser"]}, ) async def sync_role(context: dict, token: dict, userinfo: dict): + """Sync user's Grafana Org memberships based on their workspace memberships.""" + created, drycc_token = await _get_or_create_drycc_token( + userinfo["preferred_username"], token) + context["drycc_token"] = drycc_token + + workspace_orgs = await _build_workspace_orgs(userinfo, drycc_token) + async with httpx.AsyncClient() as client: - resp = await client.get(api_url("/api/user/orgs"), headers=api_headers(context, userinfo)) - orgs = resp.json() - has_admin_org = False - for org in orgs: - if org["orgId"] == 1: - has_admin_org = True - else: - context["org_id"] = org["orgId"] - if has_admin_org and not (userinfo["is_superuser"] or userinfo["is_staff"]): - await client.delete( - api_url(f"/api/orgs/1/users/{context["user_id"]}", is_admin=True), - headers=api_headers(context, userinfo), - ) - elif not has_admin_org and (userinfo["is_superuser"] or userinfo["is_staff"]): - await client.post( - api_url("/api/orgs/1/users", is_admin=True), - headers=api_headers(context, userinfo), - json={"loginOrEmail": userinfo["preferred_username"], "role": "Admin"}, - ) - await init_org(org_id=context["org_id"], name=userinfo["preferred_username"]) + resp = await client.get( + _api_url("/api/user/orgs"), headers=_api_headers(context, userinfo)) + current_orgs = resp.json() + workspace_org_ids = {info["org_id"] for info in workspace_orgs.values()} + current_org_map = {org["orgId"]: org["name"] for org in current_orgs} + + await _sync_workspace_org_memberships( + client, context, userinfo, workspace_orgs, current_org_map) + await _cleanup_stale_orgs( + client, context, userinfo, current_orgs, workspace_org_ids) + await _sync_drycc_org(client, context, userinfo, current_org_map) + + context["workspace_orgs"] = workspace_orgs async def sync_default(context: dict, token: dict, userinfo: dict): - alertmanager_default_config = json.dumps({ - "alertmanager_config": { - "route": { - "receiver": "grafana-default-email", - "group_by": ["grafana_folder", "alertname"] - }, - "receivers": [{ - "name": "grafana-default-email", - "grafana_managed_receiver_configs": [{ - "uid": "", - "name": "email receiver", - "type": "email", - "settings": { - "addresses": f"{userinfo["email"]}" - } - }] - }] - } - }) - async with httpx.AsyncClient() as client: - await client.post( - api_url("/api/folders"), - headers=api_headers(context, userinfo), - json={"uid": "drycc", "title": "drycc"}, - ) - sql_tpl = f""" - INSERT INTO alert_configuration ( - alertmanager_configuration, configuration_version, created_at, "default", org_id - ) - VALUES ( - '{alertmanager_default_config}', 'v1', {int(time.time())}, TRUE, {context["org_id"]} - ) - ON CONFLICT (org_id) DO NOTHING; + """Create default folder and alert configuration for each workspace org. + + The email receiver addresses are fully rebuilt each time based on + all workspace members with alerts=True, ensuring stale entries are removed. """ - async with await AsyncConnection.connect(os.environ.get("GF_DATABASE_URL")) as conn: - async with conn.cursor() as cursor: - await cursor.execute(sql_tpl) - await conn.commit() + workspace_orgs = context.get("workspace_orgs", {}) + drycc_token = context.get("drycc_token") + + for ws_name, ws_info in workspace_orgs.items(): + org_id = ws_info["org_id"] + alerts = ws_info["alerts"] + + alert_addresses = await _build_alert_addresses(ws_name, drycc_token, userinfo, alerts) + alertmanager_config = _build_alertmanager_config(alert_addresses) + + ctx = {**context, "org_id": org_id} + async with httpx.AsyncClient() as client: + await client.post( + _api_url("/api/folders"), + headers=_api_headers(ctx, userinfo), + json={"uid": "drycc", "title": "drycc"}, + ) + + await _upsert_alert_configuration(org_id, alertmanager_config) async def sync_alerting(context: dict, token: dict, userinfo: dict): + """Create or update alert rules for each workspace org (idempotent). + + Alert rules are always created regardless of the alerts field. + The alerts field only controls notification channels (handled in sync_default). + """ + workspace_orgs = context.get("workspace_orgs", {}) alerting_path = os.path.join(os.path.dirname(__file__), "..", "alerting") - async with httpx.AsyncClient() as client: - for filename in os.listdir(alerting_path): - with open(os.path.join(alerting_path, filename)) as f: - await client.post( - api_url("/api/v1/provisioning/alert-rules"), - headers=api_headers(context, userinfo), - json=json.load(f), + + for ws_name, ws_info in workspace_orgs.items(): + org_id = ws_info["org_id"] + ctx = {**context, "org_id": org_id} + + async with httpx.AsyncClient() as client: + for filename in os.listdir(alerting_path): + with open(os.path.join(alerting_path, filename)) as f: + rule = json.load(f) + # Use PUT for idempotent upsert (POST would create duplicates) + resp = await client.put( + _api_url(f"/api/v1/provisioning/alert-rules/{rule['uid']}"), + headers=_api_headers(ctx, userinfo), + json=rule, ) + if resp.status_code == 404: + # Rule doesn't exist yet, create it + await client.post( + _api_url("/api/v1/provisioning/alert-rules"), + headers=_api_headers(ctx, userinfo), + json=rule, + ) async def sync_datasources(context: dict, token: dict, userinfo: dict): - headers = api_headers(context, userinfo) + """Create datasources for each workspace org with workspace-specific URLs.""" + workspace_orgs = context.get("workspace_orgs", {}) datasources_path = os.path.join(os.path.dirname(__file__), "..", "datasources") - created, drycc_token = await _get_or_create_drycc_token(userinfo["preferred_username"], token) - async with httpx.AsyncClient() as client: - for filename in os.listdir(datasources_path): - with open(os.path.join(datasources_path, filename)) as f: - template = Template(f.read()) - datasource = json.loads(template.substitute( - controller_url=DRYCC_CONTROLLER_URL, - username=userinfo["preferred_username"], - time_interval=DRYCC_GRAFANA_REFRESH, - token=drycc_token - )) - resp = await client.get( - api_url(f"/api/datasources/name/{datasource["name"]}"), headers=headers) - if resp.status_code == 200: - if created: - datasource = resp.json() - datasource["secureJsonData"] = { - "httpHeaderValue1": f"Token {drycc_token}", - } - await _set_datasource_read_only(False, datasource["id"]) + drycc_token = context.get("drycc_token") + + for ws_name, ws_info in workspace_orgs.items(): + org_id = ws_info["org_id"] + ctx = {**context, "org_id": org_id} + headers = _api_headers(ctx, userinfo) + + async with httpx.AsyncClient() as client: + for filename in os.listdir(datasources_path): + with open(os.path.join(datasources_path, filename)) as f: + template = Template(f.read()) + datasource = json.loads(template.substitute( + controller_url=DRYCC_CONTROLLER_URL, + workspace=ws_name, + time_interval=DRYCC_GRAFANA_REFRESH, + token=drycc_token + )) + resp = await client.get( + _api_url(f"/api/datasources/name/{datasource['name']}"), headers=headers) + if resp.status_code == 200: + existing = resp.json() + datasource["id"] = existing["id"] + datasource["version"] = existing["version"] await client.put( - api_url(f"/api/datasources/uid/{datasource["uid"]}"), + _api_url(f"/api/datasources/uid/{datasource['uid']}"), headers=headers, json=datasource) - await _set_datasource_read_only(True, datasource["id"]) - elif resp.status_code == 404: - resp = await client.post( - api_url("/api/datasources"), headers=headers, json=datasource) - await _set_datasource_read_only(True, resp.json()["datasource"]["id"]) - else: - raise ValueError(f"grafana returned an unexpected status: {resp.status_code}") + elif resp.status_code == 404: + await client.post( + _api_url("/api/datasources"), headers=headers, json=datasource) + else: + raise ValueError( + f"grafana returned an unexpected status: {resp.status_code}" + ) async def sync_dashboards(context: dict, token: dict, userinfo: dict): + """Create dashboards for each workspace org.""" + workspace_orgs = context.get("workspace_orgs", {}) dashboards_path = os.path.join(os.path.dirname(__file__), "..", "dashboards") + + for ws_name, ws_info in workspace_orgs.items(): + org_id = ws_info["org_id"] + ctx = {**context, "org_id": org_id} + + async with httpx.AsyncClient() as client: + for filename in os.listdir(dashboards_path): + with open(os.path.join(dashboards_path, filename)) as f: + dashboard = json.load(f) + dashboard.update({"id": None, "refresh": DRYCC_GRAFANA_REFRESH}) + await client.post( + _api_url("/api/dashboards/db"), + headers=_api_headers(ctx, userinfo), + json={ + "dashboard": dashboard, + "folderUid": "drycc", + "overwrite": True, + }, + ) + + +# ── Private functions ──────────────────────────────────────────────────── + + +def _api_url(url_path, is_admin=False): + if is_admin: + return "http://{}:{}@localhost:{}{}".format( + os.environ.get('GF_SECURITY_ADMIN_USER'), + os.environ.get('GF_SECURITY_ADMIN_PASSWORD'), + os.environ.get('GF_SERVER_HTTP_PORT', 3000), + url_path, + ) + return "http://localhost:{}{}".format(os.environ.get('GF_SERVER_HTTP_PORT', 3000), url_path) + + +def _api_headers(context: dict, userinfo): + headers = {"Content-Type": "application/json"} + headers["Remote-User"] = userinfo["preferred_username"] + headers["Remote-Name"] = userinfo["preferred_username"] + headers["Remote-Email"] = userinfo["email"] + if "org_id" in context: + headers["X-Grafana-Org-Id"] = str(context["org_id"]) + return headers + + +def _get_drycc_role(userinfo: dict) -> str | None: + if userinfo["is_superuser"]: + return "Editor" + elif userinfo["is_staff"]: + return "Viewer" + return None + + +async def _get_workspaces(drycc_token: str) -> list: + """Call Controller API to get user's workspaces.""" + headers = {"Authorization": f"Token {drycc_token}"} async with httpx.AsyncClient() as client: - for filename in os.listdir(dashboards_path): - with open(os.path.join(dashboards_path, filename)) as f: - dashboard = json.load(f) - dashboard.update({"id": None, "refresh": DRYCC_GRAFANA_REFRESH}) - await client.post( - api_url("/api/dashboards/db"), - headers=api_headers(context, userinfo), - json={ - "dashboard": dashboard, - "folderUid": "drycc", - "overwrite": True, - }, - ) + resp = await client.get( + f"{DRYCC_CONTROLLER_URL}/v2/workspaces", headers=headers) + resp.raise_for_status() + return resp.json().get("results", []) + + +async def _get_workspace_members(workspace_name: str, drycc_token: str) -> list: + """Call Controller API to get workspace members.""" + headers = {"Authorization": f"Token {drycc_token}"} + async with httpx.AsyncClient() as client: + resp = await client.get( + f"{DRYCC_CONTROLLER_URL}/v2/workspaces/{workspace_name}/members", + headers=headers) + resp.raise_for_status() + return resp.json().get("results", []) + + +async def _get_or_create_org(name: str) -> int: + """Find or create a Grafana Org by name. Returns org_id.""" + headers = {"Content-Type": "application/json"} + async with httpx.AsyncClient() as client: + # Try to find existing org by name + resp = await client.get( + _api_url(f"/api/orgs/name/{name}", is_admin=True), headers=headers) + if resp.status_code == 200: + return resp.json()["id"] + # Create new org (handle race condition: another request may have created it) + resp = await client.post( + _api_url("/api/orgs", is_admin=True), headers=headers, + json={"name": name}) + if resp.status_code == 409: + # Org already exists (race condition), fetch it + resp = await client.get( + _api_url(f"/api/orgs/name/{name}", is_admin=True), headers=headers) + return resp.json()["id"] + resp.raise_for_status() + return resp.json()["orgId"] + + +async def _build_workspace_orgs(userinfo: dict, drycc_token: str) -> dict: + """Build workspace org info by fetching workspaces and their memberships. + + Returns: {workspace_name: {"org_id": int, "role": str, "alerts": bool, "email": str}} + """ + workspace_orgs = {} + try: + workspaces = await _get_workspaces(drycc_token) + except httpx.HTTPError as e: + logger.warning("Failed to fetch workspaces for %s: %s", userinfo["preferred_username"], e) + return workspace_orgs + + for ws in workspaces: + workspace_name = ws["name"] + workspace_email = ws.get("email", userinfo["email"]) + try: + members = await _get_workspace_members(workspace_name, drycc_token) + except httpx.HTTPError as e: + logger.warning("Failed to fetch members for workspace %s: %s", workspace_name, e) + continue + user_member = next( + (m for m in members if m["user"] == userinfo["preferred_username"]), None) + if user_member is None: + continue + + org_id = await _get_or_create_org(workspace_name) + workspace_orgs[workspace_name] = { + "org_id": org_id, + "role": user_member["role"], + "alerts": user_member.get("alerts", True), + "email": workspace_email, + } + return workspace_orgs + + +async def _sync_workspace_org_memberships( + client: httpx.AsyncClient, context: dict, userinfo: dict, + workspace_orgs: dict, current_org_map: dict, +): + """Add/update user's membership in each workspace org.""" + for ws_name, ws_info in workspace_orgs.items(): + org_id = ws_info["org_id"] + grafana_role = DRYCC_WORKSPACE_ROLE_MAPPING.get(ws_info["role"], "Viewer") + + if org_id in current_org_map: + await client.patch( + _api_url(f"/api/orgs/{org_id}/users/{context['user_id']}", is_admin=True), + headers=_api_headers(context, userinfo), + json={"role": grafana_role}, + ) + else: + await client.post( + _api_url(f"/api/orgs/{org_id}/users", is_admin=True), + headers=_api_headers(context, userinfo), + json={ + "loginOrEmail": userinfo["preferred_username"], + "role": grafana_role, + }, + ) + + +async def _cleanup_stale_orgs( + client: httpx.AsyncClient, context: dict, userinfo: dict, + current_orgs: list, workspace_org_ids: set, +): + """Remove user from orgs they no longer belong to (excluding drycc org).""" + for org in current_orgs: + org_id = org["orgId"] + if org_id == DRYCC_ORG_ID: + continue + if org_id not in workspace_org_ids: + await client.delete( + _api_url(f"/api/orgs/{org_id}/users/{context['user_id']}", is_admin=True), + headers=_api_headers(context, userinfo), + ) + + +async def _sync_drycc_org( + client: httpx.AsyncClient, context: dict, userinfo: dict, + current_org_map: dict, +): + """Handle drycc org membership for superusers/staff.""" + drycc_role = _get_drycc_role(userinfo) + has_drycc_org = DRYCC_ORG_ID in current_org_map + if drycc_role: + if has_drycc_org: + await client.patch( + _api_url(f"/api/orgs/{DRYCC_ORG_ID}/users/{context['user_id']}", is_admin=True), + headers=_api_headers(context, userinfo), + json={"role": drycc_role}, + ) + else: + await client.post( + _api_url(f"/api/orgs/{DRYCC_ORG_ID}/users", is_admin=True), + headers=_api_headers(context, userinfo), + json={"loginOrEmail": userinfo["preferred_username"], "role": drycc_role}, + ) + elif has_drycc_org: + await client.delete( + _api_url(f"/api/orgs/{DRYCC_ORG_ID}/users/{context['user_id']}", is_admin=True), + headers=_api_headers(context, userinfo), + ) + + +async def _build_alert_addresses( + ws_name: str, drycc_token: str, userinfo: dict, alerts: bool +) -> str: + """Build comma-separated alert email addresses for a workspace.""" + if drycc_token: + try: + members = await _get_workspace_members(ws_name, drycc_token) + email_list = [m["email"] for m in members if m.get("alerts", True)] + return ",".join(email_list) + except httpx.HTTPError as e: + logger.warning("Failed to fetch members for alert addresses in %s: %s", ws_name, e) + return userinfo["email"] if alerts else "" + return userinfo["email"] if alerts else "" + + +def _build_alertmanager_config(alert_addresses: str) -> str: + """Build alertmanager JSON config string.""" + if alert_addresses: + receivers = [{ + "name": "grafana-default-email", + "grafana_managed_receiver_configs": [{ + "uid": "", + "name": "email receiver", + "type": "email", + "settings": {"addresses": alert_addresses} + }] + }] + else: + receivers = [{ + "name": "grafana-default-email", + "grafana_managed_receiver_configs": [] + }] + return json.dumps({ + "alertmanager_config": { + "route": { + "receiver": "grafana-default-email", + "group_by": ["grafana_folder", "alertname"] + }, + "receivers": receivers + } + }) -async def _set_datasource_read_only(read_only: bool, id: str): +async def _upsert_alert_configuration(org_id: int, config: str): + """Insert or update alert configuration for an org using parameterized query.""" async with await AsyncConnection.connect(os.environ.get("GF_DATABASE_URL")) as conn: async with conn.cursor() as cursor: await cursor.execute( - "UPDATE data_source SET read_only=%s WHERE id=%s", - (read_only, id) + """ + INSERT INTO alert_configuration ( + alertmanager_configuration, configuration_version, created_at, "default", org_id + ) VALUES (%s, %s, %s, %s, %s) + ON CONFLICT (org_id) DO UPDATE + SET alertmanager_configuration = EXCLUDED.alertmanager_configuration, + configuration_version = EXCLUDED.configuration_version, + created_at = EXCLUDED.created_at + """, + (config, "v1", int(time.time()), True, org_id), ) await conn.commit() @@ -210,7 +448,7 @@ async def _check_or_create_drycc_token(drycc_token, token): if resp.status_code in [401, 403]: created = True if created: - headers = {"Authorization": f"Bearer {token["access_token"]}"} + headers = {"Authorization": f"Bearer {token['access_token']}"} data = (await client.post( f"{DRYCC_CONTROLLER_URL}/v2/auth/token/?alias=grafana-datasource", headers=headers, json=token)).json() @@ -223,7 +461,8 @@ async def _check_or_create_drycc_token(drycc_token, token): "SELECT o_auth_id_token FROM user_auth WHERE auth_module=%s AND auth_id=%s", ("authproxy", username) ) - drycc_token = (await cursor.fetchone())[0] + row = await cursor.fetchone() + drycc_token = row[0] if row else None created, drycc_token = await _check_or_create_drycc_token(drycc_token, token) if created: async with conn.cursor() as cursor: diff --git a/rootfs/usr/share/grafana/provisioning/alerting/contactpoints.yaml b/rootfs/usr/share/grafana/provisioning/alerting/contactpoints.yaml new file mode 100644 index 0000000..68f2818 --- /dev/null +++ b/rootfs/usr/share/grafana/provisioning/alerting/contactpoints.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +contactPoints: + - orgId: 1 + name: grafana-default-email + receivers: + - uid: grafana_default_email + name: email receiver + type: email + settings: + addresses: admin@drycc.cc From 67c65373011737cd59a89e01522e92d658841f36 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Mon, 11 May 2026 15:00:57 +0800 Subject: [PATCH 50/50] feat(oauth): use oauth to unify service-to-service authentication. --- .gitignore | 2 + charts/grafana/templates/_helpers.tmpl | 12 +- .../grafana/templates/grafana-configmap.yaml | 8 + .../templates/grafana-cronjob-daily.yaml | 29 ++ charts/grafana/templates/grafana-service.yaml | 4 - .../templates/grafana-statefulset.yaml | 33 ++- charts/grafana/values.yaml | 3 +- rootfs/Dockerfile | 2 +- .../grafana/oauth2/alerting/pod_cpu.json | 2 +- .../grafana/oauth2/alerting/pod_memory.json | 2 +- .../grafana/oauth2/alerting/pod_restart.json | 2 +- .../grafana/oauth2/alerting/pod_start.json | 2 +- .../grafana/oauth2/alerting/pvc_usage.json | 2 +- .../usr/share/grafana/oauth2/credentials.py | 167 ++++++++++++ .../oauth2/datasources/prometheus.json | 8 +- .../grafana/oauth2/datasources/quickwit.json | 6 +- .../usr/share/grafana/oauth2/hook/__init__.py | 4 +- .../usr/share/grafana/oauth2/hook/grafana.py | 255 +++++++++--------- rootfs/usr/share/grafana/oauth2/main.py | 159 ++++++++++- .../usr/share/grafana/oauth2/requirements.txt | 3 +- rootfs/usr/share/grafana/oauth2/settings.py | 108 ++++++++ .../provisioning/alerting/contactpoints.yaml | 11 +- .../grafana/provisioning/alerting/drycc.yaml | 30 +-- .../provisioning/datasources/drycc.yaml | 10 +- 24 files changed, 656 insertions(+), 208 deletions(-) create mode 100644 charts/grafana/templates/grafana-cronjob-daily.yaml create mode 100644 rootfs/usr/share/grafana/oauth2/credentials.py create mode 100644 rootfs/usr/share/grafana/oauth2/settings.py diff --git a/.gitignore b/.gitignore index edd7126..81b005a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ *.swp *.swo .DS_Store +.sisyphus +__pycache__ \ No newline at end of file diff --git a/charts/grafana/templates/_helpers.tmpl b/charts/grafana/templates/_helpers.tmpl index b61bb01..0519ef6 100644 --- a/charts/grafana/templates/_helpers.tmpl +++ b/charts/grafana/templates/_helpers.tmpl @@ -21,11 +21,6 @@ env: - name: DRYCC_VALKEY_URL value: "redis://:$(DRYCC_VALKEY_PASSWORD)@drycc-valkey:16379/2" {{- end }} -- name: DRYCC_SERVICE_KEY - valueFrom: - secretKeyRef: - name: controller-creds - key: service-key - name: "DRYCC_CONTROLLER_URL" value: http://drycc-controller-api - name: "DRYCC_QUICKWIT_URL" @@ -49,6 +44,11 @@ env: secretKeyRef: name: passport-creds key: drycc-passport-grafana-secret +- name: DRYCC_PASSPORT_SCOPES + valueFrom: + secretKeyRef: + name: passport-creds + key: drycc-passport-grafana-scopes {{- else }} - name: DRYCC_PASSPORT_URL value: "{{ .Values.passportUrl }}" @@ -56,6 +56,8 @@ env: value: "{{ .Values.passportKey }}" - name: DRYCC_PASSPORT_SECRET value: "{{ .Values.passportSecret }}" +- name: DRYCC_PASSPORT_SCOPES + value: "{{ .Values.passportScopes }}" {{- end }} - name: GF_DATABASE_TYPE value: postgres diff --git a/charts/grafana/templates/grafana-configmap.yaml b/charts/grafana/templates/grafana-configmap.yaml index f1c6b84..0051a8b 100644 --- a/charts/grafana/templates/grafana-configmap.yaml +++ b/charts/grafana/templates/grafana-configmap.yaml @@ -15,6 +15,14 @@ data: } } :80 { + # For security reasons, deny external access to internal endpoints. + handle /proxy/* { + respond "Not Found" 404 + } + handle /alerts/* { + respond "Not Found" 404 + } + handle /oauth2/* { reverse_proxy 127.0.0.1:4000 } diff --git a/charts/grafana/templates/grafana-cronjob-daily.yaml b/charts/grafana/templates/grafana-cronjob-daily.yaml new file mode 100644 index 0000000..8cd9a66 --- /dev/null +++ b/charts/grafana/templates/grafana-cronjob-daily.yaml @@ -0,0 +1,29 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: grafana-oauth2-token-refresher + labels: + app: grafana + component: token-refresher + heritage: drycc +spec: + schedule: "0 2 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 3 + template: + metadata: + labels: + app: grafana + component: token-refresher + spec: + restartPolicy: OnFailure + containers: + - name: token-refresher + image: {{ .Values.imageRegistry }}/{{ .Values.imageOrg }}/grafana:{{ .Values.imageTag }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + command: ["/usr/bin/env", "python3", "/usr/share/grafana/oauth2/credentials.py"] + {{- include "grafana.envs" . | indent 12 }} diff --git a/charts/grafana/templates/grafana-service.yaml b/charts/grafana/templates/grafana-service.yaml index d2e8494..b570e7a 100644 --- a/charts/grafana/templates/grafana-service.yaml +++ b/charts/grafana/templates/grafana-service.yaml @@ -25,10 +25,6 @@ spec: port: 9094 targetPort: alerting protocol: UDP - - name: oauth2-tcp - port: 4000 - targetPort: oauth2 - protocol: TCP - name: proxy-tcp port: 80 targetPort: proxy diff --git a/charts/grafana/templates/grafana-statefulset.yaml b/charts/grafana/templates/grafana-statefulset.yaml index 8a48ac2..1696c9a 100644 --- a/charts/grafana/templates/grafana-statefulset.yaml +++ b/charts/grafana/templates/grafana-statefulset.yaml @@ -158,6 +158,7 @@ spec: done echo "Grafana is up and running." python3 oauth2/main.py \ + --bind 127.0.0.1 \ --port 4000 \ --client-id $(DRYCC_PASSPORT_KEY) \ --client-secret $(DRYCC_PASSPORT_SECRET) \ @@ -167,32 +168,42 @@ spec: resources: {{- toYaml . | nindent 10 }} {{- end }} - ports: - - containerPort: 4000 - name: oauth2 {{- include "grafana.envs" . | indent 8 }} {{- if not .Values.diagnosticMode.enabled }} livenessProbe: - tcpSocket: - port: oauth2 + exec: + command: + - curl + - -fsS + - -o + - /dev/null + - http://127.0.0.1:4000/oauth2/healthz initialDelaySeconds: 240 periodSeconds: 10 timeoutSeconds: 5 failureThreshold: 6 successThreshold: 1 readinessProbe: - httpGet: - path: /oauth2/healthz - port: oauth2 + exec: + command: + - curl + - -fsS + - -o + - /dev/null + - http://127.0.0.1:4000/oauth2/healthz initialDelaySeconds: 30 periodSeconds: 10 timeoutSeconds: 5 failureThreshold: 6 successThreshold: 1 startupProbe: - httpGet: - path: /oauth2/healthz - port: oauth2 + exec: + command: + - curl + - -fsS + - -o + - /dev/null + - http://127.0.0.1:4000/oauth2/healthz initialDelaySeconds: 30 periodSeconds: 10 timeoutSeconds: 5 diff --git a/charts/grafana/values.yaml b/charts/grafana/values.yaml index 08d7b1e..9550866 100644 --- a/charts/grafana/values.yaml +++ b/charts/grafana/values.yaml @@ -64,10 +64,11 @@ environment: {} valkeyUrl: "" ## databaseUrl are will no longer use the built-in database component databaseUrl: "" -# The passportUrl, passportKey and passportSecret are will no longer use the built-in passport component +# The following parameters are used when passport.enabled is false passportUrl: "" passportKey: "" passportSecret: "" +passportScopes: "" # victoriametricsUrl is will no longer use the built-in victoriametrics component victoriametricsUrl: "" diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 1824c4d..66d795e 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -4,7 +4,7 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ARG DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/usr/share/grafana \ - GRAFANA_VERSION="12.4.2" \ + GRAFANA_VERSION="13.0.1" \ JQ_VERSION="1.7.1" \ CADDY_VERSION="2.11.2" \ PYTHON_VERSION="3.14" diff --git a/rootfs/usr/share/grafana/oauth2/alerting/pod_cpu.json b/rootfs/usr/share/grafana/oauth2/alerting/pod_cpu.json index 4e0a480..d74402f 100644 --- a/rootfs/usr/share/grafana/oauth2/alerting/pod_cpu.json +++ b/rootfs/usr/share/grafana/oauth2/alerting/pod_cpu.json @@ -74,7 +74,7 @@ }, "isPaused": false, "notification_settings": { - "receiver": "grafana-default-email" + "receiver": "controller-alerts" }, "folderUID": "drycc" } diff --git a/rootfs/usr/share/grafana/oauth2/alerting/pod_memory.json b/rootfs/usr/share/grafana/oauth2/alerting/pod_memory.json index 3eead7f..32c816d 100644 --- a/rootfs/usr/share/grafana/oauth2/alerting/pod_memory.json +++ b/rootfs/usr/share/grafana/oauth2/alerting/pod_memory.json @@ -75,7 +75,7 @@ }, "isPaused": false, "notification_settings": { - "receiver": "grafana-default-email" + "receiver": "controller-alerts" }, "folderUID": "drycc" } diff --git a/rootfs/usr/share/grafana/oauth2/alerting/pod_restart.json b/rootfs/usr/share/grafana/oauth2/alerting/pod_restart.json index ae9c144..ef1c602 100644 --- a/rootfs/usr/share/grafana/oauth2/alerting/pod_restart.json +++ b/rootfs/usr/share/grafana/oauth2/alerting/pod_restart.json @@ -77,7 +77,7 @@ "keep_firing_for": "2m", "noDataState": "OK", "notification_settings": { - "receiver": "grafana-default-email" + "receiver": "controller-alerts" }, "orgID": 1, "record": null, diff --git a/rootfs/usr/share/grafana/oauth2/alerting/pod_start.json b/rootfs/usr/share/grafana/oauth2/alerting/pod_start.json index 3fce527..119385c 100644 --- a/rootfs/usr/share/grafana/oauth2/alerting/pod_start.json +++ b/rootfs/usr/share/grafana/oauth2/alerting/pod_start.json @@ -74,7 +74,7 @@ }, "isPaused": false, "notification_settings": { - "receiver": "grafana-default-email" + "receiver": "controller-alerts" }, "folderUID": "drycc" } diff --git a/rootfs/usr/share/grafana/oauth2/alerting/pvc_usage.json b/rootfs/usr/share/grafana/oauth2/alerting/pvc_usage.json index 9aee76a..822b176 100644 --- a/rootfs/usr/share/grafana/oauth2/alerting/pvc_usage.json +++ b/rootfs/usr/share/grafana/oauth2/alerting/pvc_usage.json @@ -73,7 +73,7 @@ }, "isPaused": false, "notification_settings": { - "receiver": "grafana-default-email" + "receiver": "controller-alerts" }, "folderUID": "drycc", "ruleGroup": "middle" diff --git a/rootfs/usr/share/grafana/oauth2/credentials.py b/rootfs/usr/share/grafana/oauth2/credentials.py new file mode 100644 index 0000000..2af6af1 --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/credentials.py @@ -0,0 +1,167 @@ +import sys +import logging +import argparse +import asyncio + +import httpx +import valkey +import valkey.asyncio +import valkey.exceptions + +from settings import settings + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", +) +logger = logging.getLogger(__name__) + +# ── Constants ───────────────────────────────────────────────────── + +TOKEN_KEY = "datasource:oauth2:token" +INIT_LOCK_KEY = "datasource:oauth2:init_lock" +INIT_LOCK_TTL = 30 # Lock TTL 30 seconds +REFRESH_THRESHOLD = 7 * 24 * 3600 # Refresh when less than 7 days remain (for CronJob) + +# ── Function 1: used by main.py (get token, create if missing) ───────── + + +async def _fetch_and_save_token(valkey_client: valkey.asyncio.Valkey) -> str: + """ + Fetch a new token from Passport and save it to Valkey with TTL = expires_in. + Returns the access_token string. + """ + logger.info("Fetching new token from %s", settings.passport_token_url) + + async with httpx.AsyncClient() as client: + resp = await client.post( + settings.passport_token_url, + data={ + "grant_type": "client_credentials", + "client_id": settings.drycc_passport_key, + "client_secret": settings.drycc_passport_secret, + "scope": settings.drycc_passport_scopes, + }, + timeout=30.0, + ) + resp.raise_for_status() + token_response = resp.json() + + access_token = token_response["access_token"] + expires_in = token_response.get("expires_in", settings.session_ttl_fallback_seconds) + + await valkey_client.set(TOKEN_KEY, access_token, ex=expires_in) + + logger.info("Token saved (expires_in: %d seconds)", expires_in) + return access_token + + +async def introspect_token(token: str) -> bool: + """ + Check if the token is valid and has the required scopes via introspection. + """ + if not settings.drycc_passport_scopes: + return True + + introspect_url = settings.passport_token_url.replace("token/", "introspect/") + try: + async with httpx.AsyncClient() as client: + resp = await client.post( + introspect_url, + auth=(settings.drycc_passport_key, settings.drycc_passport_secret), + data={"token": token}, + timeout=5.0, + ) + if resp.status_code == 200: + data = resp.json() + if data.get("active"): + token_scopes = set(data.get("scope", "").split()) + required_scopes = set(settings.drycc_passport_scopes.split()) + return required_scopes == token_scopes + except Exception as e: + logger.info(f"Error introspecting token: {e}") + return False + + +async def get_token() -> str: + """ + Get the current access token. + + Flow: + 1. Fast path: read Valkey directly and return if valid + 2. Try to acquire a Valkey lock with blocking wait + 3. Double-check token and refresh if necessary + + Returns: access_token string + Raises: RuntimeError (if token cannot be obtained) + """ + valkey_client = await settings.get_valkey_client() + token = await valkey_client.get(TOKEN_KEY) + if token and await introspect_token(token): + return token + try: + async with valkey_client.lock(INIT_LOCK_KEY, timeout=30.0, blocking_timeout=30.0): + token = await valkey_client.get(TOKEN_KEY) + if token and await introspect_token(token): + return token + return await _fetch_and_save_token(valkey_client) + except valkey.exceptions.LockError: + raise RuntimeError("Timeout waiting for token refresh lock.") + + +# ── Function 2: CronJob entry point ────────────────────────────────────── + +async def main(): + """ + Asynchronous CronJob entry point. + """ + parser = argparse.ArgumentParser() + parser.add_argument( + "--force", + action="store_true", + help="Force refresh regardless of current token validity", + ) + args = parser.parse_args() + + logger.info("Token refresher started (force=%s)", args.force) + + valkey_client = await settings.get_valkey_client() + + try: + try: + await valkey_client.ping() + logger.info("Connected to Valkey") + except Exception as e: + logger.error("Failed to connect to Valkey: %s", e) + sys.exit(1) + + should_refresh = args.force + + if not should_refresh: + remaining = await valkey_client.ttl(TOKEN_KEY) + # ttl returns -2 if key missing, -1 if no expiration set + if remaining < 0: + logger.warning("No valid token found, fetching new token") + should_refresh = True + else: + remaining_days = remaining / 86400 + logger.info("Current token has %.1f days remaining", remaining_days) + if remaining > REFRESH_THRESHOLD: + logger.info("Token still valid, skip refresh") + return + logger.warning("Token expires in %.1f days, refreshing...", remaining_days) + should_refresh = True + + if should_refresh: + try: + await _fetch_and_save_token(valkey_client) + logger.info("Token refresh completed successfully") + except Exception as e: + logger.error("Token refresh failed: %s", e) + sys.exit(1) + finally: + await valkey_client.aclose() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json b/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json index aed8d47..edb10fa 100644 --- a/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json +++ b/rootfs/usr/share/grafana/oauth2/datasources/prometheus.json @@ -2,16 +2,12 @@ "name": "Prometheus on Drycc", "type": "prometheus", "uid": "prometheus_on_drycc", - "url": "${controller_url}/v2/prometheus/${workspace}", + "url": "http://localhost:4000/proxy/prometheus/${workspace}", "access": "proxy", "isDefault": true, "basicAuth": false, "jsonData": { - "httpHeaderName1": "Authorization", "httpMethod": "POST", "timeInterval": "${time_interval}" - }, - "secureJsonData": { - "httpHeaderValue1": "Token ${token}" } -} \ No newline at end of file +} diff --git a/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json b/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json index 1d1f012..682aa01 100644 --- a/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json +++ b/rootfs/usr/share/grafana/oauth2/datasources/quickwit.json @@ -2,15 +2,11 @@ "name": "Application Logs", "type": "quickwit-quickwit-datasource", "uid": "application_logs", - "url": "${controller_url}/v2/quickwit/${workspace}", + "url": "http://localhost:4000/proxy/quickwit/${workspace}", "access": "proxy", "basicAuth": false, "jsonData": { - "httpHeaderName1": "Authorization", "index": "logs-*", "logMessageField": "log" - }, - "secureJsonData": { - "httpHeaderValue1": "Token ${token}" } } diff --git a/rootfs/usr/share/grafana/oauth2/hook/__init__.py b/rootfs/usr/share/grafana/oauth2/hook/__init__.py index 5e15b49..7cbad99 100644 --- a/rootfs/usr/share/grafana/oauth2/hook/__init__.py +++ b/rootfs/usr/share/grafana/oauth2/hook/__init__.py @@ -1,7 +1,9 @@ from .grafana import ( - init_org, sync_user, sync_role, sync_default, sync_datasources, sync_dashboards, sync_alerting + has_changed, init_org, sync_user, sync_role, sync_default, sync_datasources, sync_dashboards, + sync_alerting ) +state_changed = has_changed startup_hooks = [init_org] login_hooks = [ sync_user, sync_role, sync_default, sync_datasources, sync_dashboards, sync_alerting diff --git a/rootfs/usr/share/grafana/oauth2/hook/grafana.py b/rootfs/usr/share/grafana/oauth2/hook/grafana.py index 9d25a24..cd1108c 100644 --- a/rootfs/usr/share/grafana/oauth2/hook/grafana.py +++ b/rootfs/usr/share/grafana/oauth2/hook/grafana.py @@ -1,17 +1,16 @@ import logging -import os import time import json import httpx +from pathlib import Path from string import Template from psycopg import AsyncConnection +from settings import settings logger = logging.getLogger(__name__) DEFAULT_HEADERS = {"Content-Type": "application/json"} -DRYCC_CONTROLLER_URL = os.environ.get('DRYCC_CONTROLLER_URL') -DRYCC_GRAFANA_REFRESH = os.environ.get('DRYCC_GRAFANA_REFRESH', '60s') -DRYCC_GRAFANA_DASHBOARD = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../") +DRYCC_GRAFANA_DASHBOARD = Path(__file__).resolve().parent.parent # Drycc Workspace role to Grafana role mapping DRYCC_WORKSPACE_ROLE_MAPPING = {"admin": "Editor", "member": "Editor", "viewer": "Viewer"} @@ -36,6 +35,55 @@ async def init_org(org_id=DRYCC_ORG_ID, name="drycc"): ) +async def has_changed(token: dict, userinfo: dict) -> bool: + """Detect drift between the cached userinfo / Grafana org state and the + live state in passport and the controller. + + Returns True when: + * any tracked userinfo field differs from passport's response, or + * the user's controller workspace membership count differs from the + number of workspace orgs they currently belong to in Grafana, or + * any workspace's mapped Grafana role differs from the role the user + currently has on the matching Grafana org. + Returns False when everything matches. + """ + access_token = token.get("access_token") + passport_headers = {"Authorization": f"Bearer {access_token}"} + + async with httpx.AsyncClient() as client: + passport_resp = await client.get( + settings.passport_userinfo_url, headers=passport_headers) + passport_resp.raise_for_status() + passport_userinfo = passport_resp.json() + + for field in ("preferred_username", "email", "is_superuser", "is_staff"): + if userinfo.get(field) != passport_userinfo.get(field): + return True + + workspaces = await _get_workspaces(access_token) + live_roles: dict[str, str] = {} + for ws in workspaces: + ws_name = ws["name"] + members = await _get_workspace_members(ws_name, access_token) + user_member = next( + (m for m in members if m["user"] == userinfo["preferred_username"]), None) + if user_member is None: + continue + live_roles[ws_name] = DRYCC_WORKSPACE_ROLE_MAPPING.get( + user_member["role"], "Viewer") + + grafana_resp = await client.get( + _api_url("/api/user/orgs"), headers=_api_headers({}, userinfo)) + grafana_resp.raise_for_status() + grafana_roles = { + org["name"]: org["role"] + for org in grafana_resp.json() + if org["name"] != "drycc" + } + + return live_roles != grafana_roles + + async def sync_user(context: dict, token: dict, userinfo: dict): async with httpx.AsyncClient() as client: resp = await client.get(_api_url("/api/user"), headers=_api_headers(context, userinfo)) @@ -51,11 +99,10 @@ async def sync_user(context: dict, token: dict, userinfo: dict): async def sync_role(context: dict, token: dict, userinfo: dict): """Sync user's Grafana Org memberships based on their workspace memberships.""" - created, drycc_token = await _get_or_create_drycc_token( - userinfo["preferred_username"], token) - context["drycc_token"] = drycc_token + access_token = token.get("access_token") + context["access_token"] = access_token - workspace_orgs = await _build_workspace_orgs(userinfo, drycc_token) + workspace_orgs = await _build_workspace_orgs(userinfo, access_token) async with httpx.AsyncClient() as client: resp = await client.get( @@ -76,18 +123,16 @@ async def sync_role(context: dict, token: dict, userinfo: dict): async def sync_default(context: dict, token: dict, userinfo: dict): """Create default folder and alert configuration for each workspace org. - The email receiver addresses are fully rebuilt each time based on - all workspace members with alerts=True, ensuring stale entries are removed. + Each workspace org gets a webhook contact point pointing back to this + service's /alerts/webhook endpoint, which fans out to passport messages + for workspace members with alerts=True. """ workspace_orgs = context.get("workspace_orgs", {}) - drycc_token = context.get("drycc_token") for ws_name, ws_info in workspace_orgs.items(): org_id = ws_info["org_id"] - alerts = ws_info["alerts"] - alert_addresses = await _build_alert_addresses(ws_name, drycc_token, userinfo, alerts) - alertmanager_config = _build_alertmanager_config(alert_addresses) + alertmanager_config = _build_alertmanager_config(ws_name) ctx = {**context, "org_id": org_id} async with httpx.AsyncClient() as client: @@ -107,15 +152,15 @@ async def sync_alerting(context: dict, token: dict, userinfo: dict): The alerts field only controls notification channels (handled in sync_default). """ workspace_orgs = context.get("workspace_orgs", {}) - alerting_path = os.path.join(os.path.dirname(__file__), "..", "alerting") + alerting_path = Path(__file__).resolve().parent.parent / "alerting" - for ws_name, ws_info in workspace_orgs.items(): + for _, ws_info in workspace_orgs.items(): org_id = ws_info["org_id"] ctx = {**context, "org_id": org_id} async with httpx.AsyncClient() as client: - for filename in os.listdir(alerting_path): - with open(os.path.join(alerting_path, filename)) as f: + for filepath in alerting_path.glob("*.json"): + with filepath.open() as f: rule = json.load(f) # Use PUT for idempotent upsert (POST would create duplicates) resp = await client.put( @@ -133,38 +178,47 @@ async def sync_alerting(context: dict, token: dict, userinfo: dict): async def sync_datasources(context: dict, token: dict, userinfo: dict): - """Create datasources for each workspace org with workspace-specific URLs.""" + """Create datasources for each workspace org with workspace-specific URLs. + + Datasource creation requires Org Admin permission. Workspace users are + only Editor/Viewer, so we must use Grafana admin basic auth combined with + X-Grafana-Org-Id to write into the target workspace org. + """ workspace_orgs = context.get("workspace_orgs", {}) - datasources_path = os.path.join(os.path.dirname(__file__), "..", "datasources") - drycc_token = context.get("drycc_token") + datasources_path = Path(__file__).resolve().parent.parent / "datasources" for ws_name, ws_info in workspace_orgs.items(): org_id = ws_info["org_id"] - ctx = {**context, "org_id": org_id} - headers = _api_headers(ctx, userinfo) + headers = { + "Content-Type": "application/json", + "X-Grafana-Org-Id": str(org_id), + } async with httpx.AsyncClient() as client: - for filename in os.listdir(datasources_path): - with open(os.path.join(datasources_path, filename)) as f: + for filepath in datasources_path.glob("*.json"): + with filepath.open() as f: template = Template(f.read()) datasource = json.loads(template.substitute( - controller_url=DRYCC_CONTROLLER_URL, + controller_url=settings.controller_base_url, + time_interval=settings.drycc_grafana_refresh, workspace=ws_name, - time_interval=DRYCC_GRAFANA_REFRESH, - token=drycc_token )) resp = await client.get( - _api_url(f"/api/datasources/name/{datasource['name']}"), headers=headers) + _api_url(f"/api/datasources/name/{datasource['name']}", is_admin=True), + headers=headers) if resp.status_code == 200: existing = resp.json() datasource["id"] = existing["id"] datasource["version"] = existing["version"] - await client.put( - _api_url(f"/api/datasources/uid/{datasource['uid']}"), + resp = await client.put( + _api_url(f"/api/datasources/uid/{datasource['uid']}", is_admin=True), headers=headers, json=datasource) + resp.raise_for_status() elif resp.status_code == 404: - await client.post( - _api_url("/api/datasources"), headers=headers, json=datasource) + resp = await client.post( + _api_url("/api/datasources", is_admin=True), + headers=headers, json=datasource) + resp.raise_for_status() else: raise ValueError( f"grafana returned an unexpected status: {resp.status_code}" @@ -174,17 +228,17 @@ async def sync_datasources(context: dict, token: dict, userinfo: dict): async def sync_dashboards(context: dict, token: dict, userinfo: dict): """Create dashboards for each workspace org.""" workspace_orgs = context.get("workspace_orgs", {}) - dashboards_path = os.path.join(os.path.dirname(__file__), "..", "dashboards") + dashboards_path = Path(__file__).resolve().parent.parent / "dashboards" for ws_name, ws_info in workspace_orgs.items(): org_id = ws_info["org_id"] ctx = {**context, "org_id": org_id} async with httpx.AsyncClient() as client: - for filename in os.listdir(dashboards_path): - with open(os.path.join(dashboards_path, filename)) as f: + for filepath in dashboards_path.glob("*.json"): + with filepath.open() as f: dashboard = json.load(f) - dashboard.update({"id": None, "refresh": DRYCC_GRAFANA_REFRESH}) + dashboard.update({"id": None, "refresh": settings.drycc_grafana_refresh}) await client.post( _api_url("/api/dashboards/db"), headers=_api_headers(ctx, userinfo), @@ -202,12 +256,12 @@ async def sync_dashboards(context: dict, token: dict, userinfo: dict): def _api_url(url_path, is_admin=False): if is_admin: return "http://{}:{}@localhost:{}{}".format( - os.environ.get('GF_SECURITY_ADMIN_USER'), - os.environ.get('GF_SECURITY_ADMIN_PASSWORD'), - os.environ.get('GF_SERVER_HTTP_PORT', 3000), - url_path, + settings.gf_security_admin_user, + settings.gf_security_admin_password, + settings.gf_server_http_port, + url_path ) - return "http://localhost:{}{}".format(os.environ.get('GF_SERVER_HTTP_PORT', 3000), url_path) + return "http://localhost:{}{}".format(settings.gf_server_http_port, url_path) def _api_headers(context: dict, userinfo): @@ -228,22 +282,22 @@ def _get_drycc_role(userinfo: dict) -> str | None: return None -async def _get_workspaces(drycc_token: str) -> list: +async def _get_workspaces(access_token: str) -> list: """Call Controller API to get user's workspaces.""" - headers = {"Authorization": f"Token {drycc_token}"} + headers = {"Authorization": f"Bearer {access_token}"} async with httpx.AsyncClient() as client: resp = await client.get( - f"{DRYCC_CONTROLLER_URL}/v2/workspaces", headers=headers) + f"{settings.controller_base_url}/v2/workspaces", headers=headers) resp.raise_for_status() return resp.json().get("results", []) -async def _get_workspace_members(workspace_name: str, drycc_token: str) -> list: +async def _get_workspace_members(workspace_id: str, access_token: str) -> list: """Call Controller API to get workspace members.""" - headers = {"Authorization": f"Token {drycc_token}"} + headers = {"Authorization": f"Bearer {access_token}"} async with httpx.AsyncClient() as client: resp = await client.get( - f"{DRYCC_CONTROLLER_URL}/v2/workspaces/{workspace_name}/members", + f"{settings.controller_base_url}/v2/workspaces/{workspace_id}/members", headers=headers) resp.raise_for_status() return resp.json().get("results", []) @@ -271,37 +325,34 @@ async def _get_or_create_org(name: str) -> int: return resp.json()["orgId"] -async def _build_workspace_orgs(userinfo: dict, drycc_token: str) -> dict: +async def _build_workspace_orgs(userinfo: dict, access_token: str) -> dict: """Build workspace org info by fetching workspaces and their memberships. - Returns: {workspace_name: {"org_id": int, "role": str, "alerts": bool, "email": str}} + Returns: {workspace_id: {"org_id": int, "role": str}} """ workspace_orgs = {} try: - workspaces = await _get_workspaces(drycc_token) + workspaces = await _get_workspaces(access_token) except httpx.HTTPError as e: logger.warning("Failed to fetch workspaces for %s: %s", userinfo["preferred_username"], e) return workspace_orgs for ws in workspaces: - workspace_name = ws["name"] - workspace_email = ws.get("email", userinfo["email"]) + workspace_id = ws["id"] try: - members = await _get_workspace_members(workspace_name, drycc_token) + members = await _get_workspace_members(workspace_id, access_token) except httpx.HTTPError as e: - logger.warning("Failed to fetch members for workspace %s: %s", workspace_name, e) + logger.warning("Failed to fetch members for workspace %s: %s", workspace_id, e) continue user_member = next( (m for m in members if m["user"] == userinfo["preferred_username"]), None) if user_member is None: continue - org_id = await _get_or_create_org(workspace_name) - workspace_orgs[workspace_name] = { + org_id = await _get_or_create_org(workspace_id) + workspace_orgs[workspace_id] = { "org_id": org_id, "role": user_member["role"], - "alerts": user_member.get("alerts", True), - "email": workspace_email, } return workspace_orgs @@ -375,42 +426,23 @@ async def _sync_drycc_org( ) -async def _build_alert_addresses( - ws_name: str, drycc_token: str, userinfo: dict, alerts: bool -) -> str: - """Build comma-separated alert email addresses for a workspace.""" - if drycc_token: - try: - members = await _get_workspace_members(ws_name, drycc_token) - email_list = [m["email"] for m in members if m.get("alerts", True)] - return ",".join(email_list) - except httpx.HTTPError as e: - logger.warning("Failed to fetch members for alert addresses in %s: %s", ws_name, e) - return userinfo["email"] if alerts else "" - return userinfo["email"] if alerts else "" - - -def _build_alertmanager_config(alert_addresses: str) -> str: - """Build alertmanager JSON config string.""" - if alert_addresses: - receivers = [{ - "name": "grafana-default-email", - "grafana_managed_receiver_configs": [{ - "uid": "", - "name": "email receiver", - "type": "email", - "settings": {"addresses": alert_addresses} - }] - }] - else: - receivers = [{ - "name": "grafana-default-email", - "grafana_managed_receiver_configs": [] - }] +def _build_alertmanager_config(ws_name: str) -> str: + receivers = [{ + "name": "controller-alerts", + "grafana_managed_receiver_configs": [{ + "uid": "", + "name": "controller alerts webhook", + "type": "webhook", + "settings": { + "url": f"http://localhost:4000/alerts/webhook?workspace={ws_name}", + "httpMethod": "POST", + }, + }], + }] return json.dumps({ "alertmanager_config": { "route": { - "receiver": "grafana-default-email", + "receiver": "controller-alerts", "group_by": ["grafana_folder", "alertname"] }, "receivers": receivers @@ -420,7 +452,7 @@ def _build_alertmanager_config(alert_addresses: str) -> str: async def _upsert_alert_configuration(org_id: int, config: str): """Insert or update alert configuration for an org using parameterized query.""" - async with await AsyncConnection.connect(os.environ.get("GF_DATABASE_URL")) as conn: + async with await AsyncConnection.connect(settings.gf_database_url) as conn: async with conn.cursor() as cursor: await cursor.execute( """ @@ -435,40 +467,3 @@ async def _upsert_alert_configuration(org_id: int, config: str): (config, "v1", int(time.time()), True, org_id), ) await conn.commit() - - -async def _get_or_create_drycc_token(username, token: dict): - async def _check_or_create_drycc_token(drycc_token, token): - async with httpx.AsyncClient() as client: - created = False if drycc_token else True - if drycc_token: - headers = {"Authorization": f"Token {drycc_token}"} - resp = await client.get( - f"{DRYCC_CONTROLLER_URL}/v2/auth/whoami", headers=headers) - if resp.status_code in [401, 403]: - created = True - if created: - headers = {"Authorization": f"Bearer {token['access_token']}"} - data = (await client.post( - f"{DRYCC_CONTROLLER_URL}/v2/auth/token/?alias=grafana-datasource", - headers=headers, json=token)).json() - drycc_token = data["token"] - return created, drycc_token - - async with await AsyncConnection.connect(os.environ.get("GF_DATABASE_URL")) as conn: - async with conn.cursor() as cursor: - await cursor.execute( - "SELECT o_auth_id_token FROM user_auth WHERE auth_module=%s AND auth_id=%s", - ("authproxy", username) - ) - row = await cursor.fetchone() - drycc_token = row[0] if row else None - created, drycc_token = await _check_or_create_drycc_token(drycc_token, token) - if created: - async with conn.cursor() as cursor: - await cursor.execute( - "UPDATE user_auth SET o_auth_id_token=%s WHERE auth_module=%s AND auth_id=%s", - (drycc_token, "authproxy", username) - ) - await conn.commit() - return created, drycc_token diff --git a/rootfs/usr/share/grafana/oauth2/main.py b/rootfs/usr/share/grafana/oauth2/main.py index 4973911..0726ba4 100644 --- a/rootfs/usr/share/grafana/oauth2/main.py +++ b/rootfs/usr/share/grafana/oauth2/main.py @@ -1,13 +1,18 @@ +import json import random import string import argparse +import uuid +import httpx from contextlib import asynccontextmanager -from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse, RedirectResponse +from fastapi import FastAPI, Request, HTTPException +from fastapi.responses import JSONResponse, RedirectResponse, Response, StreamingResponse from authlib.integrations.starlette_client import OAuth from starlette.middleware.sessions import SessionMiddleware -from hook import startup_hooks, login_hooks, destroy_hooks +from hook import state_changed, startup_hooks, login_hooks, destroy_hooks +from settings import settings +from credentials import get_token def randstr(k=32): @@ -25,11 +30,26 @@ def randstr(k=32): args = parser.parse_args() +http_client = None + +SESSION_KEY_PREFIX = "oauth2:session:" +STATE_CHECK_KEY_PREFIX = "oauth2:state_check:" + + @asynccontextmanager async def lifespan(app: FastAPI): + global http_client + http_client = httpx.AsyncClient(timeout=30.0) + + try: + await get_token() + except Exception as e: + raise RuntimeError(f"Failed to initialize token: {e}") + for startup_hook in startup_hooks: await startup_hook() yield + await http_client.aclose() for destroy_hook in destroy_hooks: await destroy_hook() @@ -55,10 +75,17 @@ async def healthz(): @app.get("/oauth2/sign_in") async def oauth2_sign_in(request: Request): - if "user" in request.session: - return RedirectResponse(request.query_params.get("redirect", "/")) - redirect_url = request.url_for("oauth2_callback") - redirect_url.replace_query_params(redirect=request.query_params.get("redirect", "/")) + sid = request.session.get("sid") + redirect_to = request.query_params.get("redirect", "/") + if sid: + valkey_client = await settings.get_valkey_client() + try: + key = f"{SESSION_KEY_PREFIX}{sid}" + if await valkey_client.get(key): + return RedirectResponse(redirect_to) + finally: + await valkey_client.aclose() + redirect_url = request.url_for("oauth2_callback").replace_query_params(redirect=redirect_to) return await oauth.oidc.authorize_redirect(request, redirect_url) @@ -70,7 +97,18 @@ async def oauth2_callback(request: Request): else: token.pop("id_token", None) userinfo = token.pop("userinfo") - request.session['user'] = userinfo + + sid = uuid.uuid4().hex + valkey_client = await settings.get_valkey_client() + try: + key = f"{SESSION_KEY_PREFIX}{sid}" + payload = json.dumps({"token": token, "user": userinfo}) + expires_in = token.get("expires_in") or settings.session_ttl_fallback_seconds + await valkey_client.set(key, payload, ex=expires_in) + finally: + await valkey_client.aclose() + request.session['sid'] = sid + context = {} for login_hook in login_hooks: await login_hook(context, token, userinfo) @@ -79,9 +117,36 @@ async def oauth2_callback(request: Request): @app.get("/oauth2/userinfo") async def oauth2_userinfo(request: Request): - if "user" not in request.session: + sid = request.session.get("sid") + if not sid: return JSONResponse(status_code=401, content={"error": "Unauthorized"}) - userinfo = request.session['user'] + + session_key = f"{SESSION_KEY_PREFIX}{sid}" + state_check_key = f"{STATE_CHECK_KEY_PREFIX}{sid}" + valkey_client = await settings.get_valkey_client() + try: + raw = await valkey_client.get(session_key) + if not raw: + request.session.pop("sid", None) + return JSONResponse(status_code=401, content={"error": "Unauthorized"}) + + payload = json.loads(raw) + token, userinfo = payload["token"], payload["user"] + + # SET NX EX guarantees only one request per cooldown window runs the + # expensive upstream check, even when Grafana fans out concurrent + # /oauth2/userinfo calls. + cooldown = settings.state_check_cooldown_seconds + check_due = cooldown <= 0 or bool( + await valkey_client.set(state_check_key, "1", ex=cooldown, nx=True) + ) + if check_due and await state_changed(token, userinfo): + await valkey_client.delete(session_key) + request.session.pop("sid", None) + return JSONResponse(status_code=401, content={"error": "Unauthorized"}) + finally: + await valkey_client.aclose() + headers = { "Remote-User": userinfo['preferred_username'], "Remote-Name": userinfo['preferred_username'], @@ -90,6 +155,80 @@ async def oauth2_userinfo(request: Request): return JSONResponse(content=userinfo, headers=headers) +# ── Proxy routes (zero-overhead version) ─────────────────────────── + +async def _proxy_request(request: Request, url: str): + """Minimal proxy: get the token directly from token.py and use it.""" + try: + token = await get_token() + except RuntimeError as e: + raise HTTPException(status_code=503, detail=str(e)) + + headers = dict(request.headers) + headers.pop("host", None) + headers["Authorization"] = f"Bearer {token}" + resp = await http_client.request( + method=request.method, url=url, headers=headers, params=request.query_params, + content=await request.body(), + ) + return StreamingResponse( + resp.aiter_bytes(), status_code=resp.status_code, headers=dict(resp.headers) + ) + + +@app.api_route( + "/proxy/prometheus/{workspace}/{path:path}", + methods=["GET", "POST", "PUT", "DELETE"], +) +async def proxy_prometheus(workspace: str, path: str, request: Request): + url = f"{settings.controller_base_url}/v2/prometheus/{workspace}/{path}" + return await _proxy_request(request, url) + + +@app.api_route( + "/proxy/quickwit/{workspace}/{path:path}", + methods=["GET", "POST", "PUT", "DELETE"], +) +async def proxy_quickwit(workspace: str, path: str, request: Request): + url = f"{settings.controller_base_url}/v2/quickwit/{workspace}/{path}" + return await _proxy_request(request, url) + + +# ── Alert webhook (translate Grafana payload → controller schema) ──── + + +@app.post("/alerts/webhook") +async def alerts_webhook(workspace: str, request: Request): + def _translate_alert(alert: dict) -> dict: + _SEVERITY_CHOICES = {"info", "warning", "error", "success"} + _STATUS_TO_SEVERITY = {"firing": "warning", "resolved": "success"} + labels, annotations = alert.get("labels") or {}, alert.get("annotations") or {} + title = (annotations.get("summary") or labels.get("alertname") or "Grafana alert")[:255] + content = annotations.get("description") or title + severity = (labels.get("severity") if labels.get("severity") in _SEVERITY_CHOICES + else _STATUS_TO_SEVERITY.get(alert.get("status", "firing"), "info")) + message = {"category": "alert", "title": title, "content": content, "severity": severity} + generator_url = alert.get("generatorURL") + if generator_url: + message.update({"action_link": generator_url[:500], "action_text": "View in Grafana"}) + return message + + try: + token = await get_token() + except RuntimeError as e: + raise HTTPException(status_code=503, detail=str(e)) + grafana_payload = await request.json() + alerts = (grafana_payload or {}).get("alerts") or [] + translated = [_translate_alert(a) for a in alerts] + body = json.dumps({"workspace": workspace, "alerts": translated}) + resp = await http_client.post( + f"{settings.controller_base_url}/v2/alerts", + headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}, + content=body, + ) + return Response(status_code=resp.status_code) + + if __name__ == "__main__": import uvicorn uvicorn.run(app, host=args.bind, port=int(args.port)) diff --git a/rootfs/usr/share/grafana/oauth2/requirements.txt b/rootfs/usr/share/grafana/oauth2/requirements.txt index 4061b5f..11d9153 100644 --- a/rootfs/usr/share/grafana/oauth2/requirements.txt +++ b/rootfs/usr/share/grafana/oauth2/requirements.txt @@ -1,5 +1,5 @@ fastapi>=0.68.0 -redis[asyncio]>=6.2.0 +valkey>=6.1.0 uvicorn[standard]>=0.15.0 authlib>=1.0.0 httpx>=0.23.0 @@ -7,3 +7,4 @@ psycopg[binary]>=3.2.9 python-jose>=3.3.0 python-multipart>=0.0.5 itsdangerous>=2.2.0 +pydantic_settings>=2.14.1 \ No newline at end of file diff --git a/rootfs/usr/share/grafana/oauth2/settings.py b/rootfs/usr/share/grafana/oauth2/settings.py new file mode 100644 index 0000000..8fab532 --- /dev/null +++ b/rootfs/usr/share/grafana/oauth2/settings.py @@ -0,0 +1,108 @@ +import re + +import valkey.asyncio +from pydantic import Field, computed_field, RedisDsn, HttpUrl +from pydantic_settings import BaseSettings, SettingsConfigDict + + +_DURATION_RE = re.compile(r"^\s*(\d+)\s*([smhd]?)\s*$", re.IGNORECASE) +_DURATION_MULTIPLIERS = {"": 1, "s": 1, "m": 60, "h": 3600, "d": 86400} + + +def _parse_duration(value: str) -> int: + """Parse a duration string like '60s', '5m', '1h', '7d' into seconds. + + A bare integer (no unit) is treated as seconds. Raises ValueError on + malformed input so misconfiguration fails fast at startup. + """ + if value is None: + raise ValueError("duration value must not be None") + match = _DURATION_RE.match(str(value)) + if not match: + raise ValueError(f"invalid duration: {value!r}") + number, unit = match.groups() + return int(number) * _DURATION_MULTIPLIERS[unit.lower()] + + +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_prefix="", + extra="ignore", + case_sensitive=True, + ) + + # Drycc configuration + drycc_valkey_url: RedisDsn = Field( + default="redis://localhost:6379/0", + validation_alias="DRYCC_VALKEY_URL", + ) + drycc_passport_url: HttpUrl = Field( + default="http://passport.drycc.cc", + validation_alias="DRYCC_PASSPORT_URL", + ) + drycc_passport_key: str = Field(default="", validation_alias="DRYCC_PASSPORT_KEY") + drycc_passport_secret: str = Field(default="", validation_alias="DRYCC_PASSPORT_SECRET") + drycc_passport_scopes: str = Field(default="", validation_alias="DRYCC_PASSPORT_SCOPES") + drycc_controller_url: HttpUrl = Field( + default="http://controller.drycc.cc", + validation_alias="DRYCC_CONTROLLER_URL", + ) + drycc_grafana_refresh: str = Field(default="60s", validation_alias="DRYCC_GRAFANA_REFRESH") + + drycc_state_check_cooldown: str = Field( + default="60s", + validation_alias="DRYCC_STATE_CHECK_COOLDOWN", + ) + drycc_session_ttl_fallback: str = Field( + default="7d", + validation_alias="DRYCC_SESSION_TTL_FALLBACK", + ) + + # Grafana configuration + gf_security_admin_user: str = Field(default="admin", validation_alias="GF_SECURITY_ADMIN_USER") + gf_security_admin_password: str = Field( + default="admin", + validation_alias="GF_SECURITY_ADMIN_PASSWORD", + ) + gf_server_http_port: int = Field(default=3000, validation_alias="GF_SERVER_HTTP_PORT") + gf_database_url: str = Field(default="", validation_alias="GF_DATABASE_URL") + + @computed_field + @property + def passport_token_url(self) -> str: + return f"{str(self.drycc_passport_url).rstrip('/')}/oauth/token/" + + @computed_field + @property + def passport_userinfo_url(self) -> str: + return f"{str(self.drycc_passport_url).rstrip('/')}/oauth/userinfo/" + + @computed_field + @property + def controller_base_url(self) -> str: + """Controller URL normalized without trailing slash. + + Pydantic's HttpUrl appends a trailing slash on serialization, which + causes double slashes when callers concatenate paths with a leading + slash (e.g. f"{url}/v2/..."). Use this property for any path joining. + """ + return str(self.drycc_controller_url).rstrip('/') + + @computed_field + @property + def state_check_cooldown_seconds(self) -> int: + return _parse_duration(self.drycc_state_check_cooldown) + + @computed_field + @property + def session_ttl_fallback_seconds(self) -> int: + return _parse_duration(self.drycc_session_ttl_fallback) + + async def get_valkey_client(self) -> valkey.asyncio.Valkey: + return valkey.asyncio.from_url( + str(self.drycc_valkey_url), + decode_responses=True, + ) + + +settings = Settings() diff --git a/rootfs/usr/share/grafana/provisioning/alerting/contactpoints.yaml b/rootfs/usr/share/grafana/provisioning/alerting/contactpoints.yaml index 68f2818..e217bd0 100644 --- a/rootfs/usr/share/grafana/provisioning/alerting/contactpoints.yaml +++ b/rootfs/usr/share/grafana/provisioning/alerting/contactpoints.yaml @@ -2,10 +2,11 @@ apiVersion: 1 contactPoints: - orgId: 1 - name: grafana-default-email + name: controller-alerts receivers: - - uid: grafana_default_email - name: email receiver - type: email + - uid: controller_alerts + name: controller alerts webhook + type: webhook settings: - addresses: admin@drycc.cc + url: http://localhost:4000/alerts/webhook?workspace=drycc + httpMethod: POST diff --git a/rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml b/rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml index a4f1545..881813d 100644 --- a/rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml +++ b/rootfs/usr/share/grafana/provisioning/alerting/drycc.yaml @@ -57,7 +57,7 @@ groups: summary: Pod restart alert isPaused: false notification_settings: - receiver: grafana-default-email + receiver: controller-alerts - uid: deq6i5dvnlkw0e title: pod cpu usage over 80% condition: C @@ -109,7 +109,7 @@ groups: summary: CPU Usage of Pod limit on has exceeded 80%. isPaused: false notification_settings: - receiver: grafana-default-email + receiver: controller-alerts - uid: beq6l7fts9nnke title: pod memory usage over 80% condition: C @@ -162,7 +162,7 @@ groups: summary: Memory Usage of Pod limit on has exceeded 80%. isPaused: false notification_settings: - receiver: grafana-default-email + receiver: controller-alerts - uid: eeq6lm6biyfb4e title: pod start fail condition: C @@ -214,7 +214,7 @@ groups: summary: Pod start fail isPaused: false notification_settings: - receiver: grafana-default-email + receiver: controller-alerts - uid: feq6m6yr27togc title: job exec error condition: C @@ -266,7 +266,7 @@ groups: summary: Job exec error isPaused: false notification_settings: - receiver: grafana-default-email + receiver: controller-alerts - uid: beqhp6i29jx8gb title: pvc usage over 80% condition: C @@ -318,7 +318,7 @@ groups: summary: Pvc usage over 80% isPaused: false notification_settings: - receiver: grafana-default-email + receiver: controller-alerts - uid: deqhpblphb7k0c title: mysql down condition: C @@ -370,7 +370,7 @@ groups: summary: MySQL not up. isPaused: false notification_settings: - receiver: grafana-default-email + receiver: controller-alerts - uid: feqhpe2zz56o0d title: mysql group replcation down condition: C @@ -422,7 +422,7 @@ groups: summary: MySQL group replication not up. isPaused: false notification_settings: - receiver: grafana-default-email + receiver: controller-alerts - orgId: 1 name: high folder: drycc @@ -479,7 +479,7 @@ groups: summary: Node down isPaused: false notification_settings: - receiver: grafana-default-email + receiver: controller-alerts - uid: eeq6lu5wz3eo0c title: cluster node over pressure condition: C @@ -531,7 +531,7 @@ groups: summary: Cluster node over pressure isPaused: false notification_settings: - receiver: grafana-default-email + receiver: controller-alerts - uid: deq6lwic91slce title: cluster node status error condition: C @@ -583,7 +583,7 @@ groups: summary: Cluster node status error isPaused: false notification_settings: - receiver: grafana-default-email + receiver: controller-alerts - orgId: 1 name: middle folder: drycc @@ -640,7 +640,7 @@ groups: summary: Node cpu high over 80 isPaused: false notification_settings: - receiver: grafana-default-email + receiver: controller-alerts - uid: ceqhohsqqyr5sb title: node memory usage over 80% condition: C @@ -692,7 +692,7 @@ groups: summary: Node memory over 80 isPaused: false notification_settings: - receiver: grafana-default-email + receiver: controller-alerts - uid: aeqhp0wbcsb9cf title: node systen disk usage over 80% condition: C @@ -744,7 +744,7 @@ groups: summary: Node systen disk usage over 80 isPaused: false notification_settings: - receiver: grafana-default-email + receiver: controller-alerts - orgId: 1 name: slow folder: drycc @@ -800,4 +800,4 @@ groups: summary: Node certificate expiration isPaused: false notification_settings: - receiver: grafana-default-email + receiver: controller-alerts diff --git a/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml b/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml index d9b9a66..85983ee 100644 --- a/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml +++ b/rootfs/usr/share/grafana/provisioning/datasources/drycc.yaml @@ -5,13 +5,10 @@ datasources: access: proxy orgId: 1 uid: application_logs - url: $DRYCC_QUICKWIT_URL + url: http://localhost:4000/proxy/quickwit/drycc jsonData: - httpHeaderName1: X-Drycc-Service-Key index: logs-* logMessageField: log - secureJsonData: - httpHeaderValue1: $DRYCC_SERVICE_KEY version: 1 editable: false @@ -20,12 +17,9 @@ datasources: access: proxy orgId: 1 uid: prometheus_on_drycc - url: $DRYCC_VICTORIAMETRICS_URL + url: http://localhost:4000/proxy/prometheus/drycc isDefault: true jsonData: httpMethod: POST - httpHeaderName1: X-Drycc-Service-Key - secureJsonData: - httpHeaderValue1: $DRYCC_SERVICE_KEY version: 1 editable: false