Skip to content

Commit 0a572d5

Browse files
duanhongyiduanhongyi
committed
chore(monitor): add postgresql outputs
1 parent d898b65 commit 0a572d5

7 files changed

Lines changed: 83 additions & 340 deletions

File tree

‎README.md‎

Lines changed: 26 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -15,36 +15,36 @@ This repository aims to contain the necessary components for a production qualit
1515
## Current State
1616
Currently this repo provides only 2 components (Telegraf, and Grafana). Telegraf is the metrics collection agent that runs using the daemon set API. For more infomation please read [this](telegraf/README.md).
1717

18-
Grafana is a stand alone graphing application. It natively supports Influxdb as a datasource and provides a robust engine for creating dashboards on top of timeseries data. We provide a few out of the box dashboards for monitoring Drycc Workflow and Kubernetes but please feel free to use them as a starting point for creating your own dashboards.
18+
Grafana is a stand alone graphing application. It natively supports Timescaledb as a datasource and provides a robust engine for creating dashboards on top of timeseries data. We provide a few out of the box dashboards for monitoring Drycc Workflow and Kubernetes but please feel free to use them as a starting point for creating your own dashboards.
1919

2020
# Architecture Diagram
2121

2222
```
23-
┌────────┐ ┌────────┐
24-
│ Router │ │ Logger │
25-
└────────┘ └────────┘
26-
│ │
27-
logs file │
28-
│ │
29-
▼ ▼
30-
┌──────────┐ ┌─────────┐ logs/metrics ┌──────────────┐
31-
│ App Logs │──Log File──▶│ fluentd │─────topics───▶│ Redis Stream │
32-
└──────────┘ └─────────┘ └──────────────┘
33-
│
34-
┌──────────┐ │
35-
│ HOST │───┐ │
36-
│ Telegraf │ │ │
37-
└──────────┘ │ │
38-
│ ▼
39-
┌──────────┐ │ ┌──────────┐ ┌──────────┐
40-
│ HOST │───┼───▶│ Influxdb │◀────────Wire────────│ Telegraf │
41-
│ Telegraf │ │ └──────────┘ Protocol └──────────┘
42-
└──────────┘ │ │
43-
│ │
44-
┌──────────┐ │ ▼
45-
│ HOST │───┘ ┌─────────┐ ┌────────────┐
46-
│ Telegraf │ │ Grafana │◀────────────────────│ Prometheus │
47-
└──────────┘ └─────────┘ └────────────┘
23+
┌─────────────┐ ┌──────────────┐
24+
│ Router │ │ Logger │
25+
└─────────────┘ └──────────────┘
26+
│ │
27+
logs file │
28+
│ │
29+
▼ ▼
30+
┌──────────┐ ┌───-─────────┐ logs/metrics ┌──────────────┐
31+
│ App Logs │───Log File───▶│ Fluentd │────topics──────▶│ Redis Stream │
32+
└──────────┘ └────-────────┘ └──────────────┘
33+
│
34+
┌──────────┐ │
35+
│ HOST │───┐ │
36+
│ Telegraf │ │ │
37+
└──────────┘ │ │
38+
│ ▼
39+
┌──────────┐ │ ┌───────-─────┐ ┌─────────────┐
40+
│ HOST │───┼──────────▶│ Timescaledb │◀──────Wire──────│ Telegraf │
41+
│ Telegraf │ │ └───────────-─┘ Protocol └─────────────┘
42+
└──────────┘ │ │
43+
│ │
44+
┌──────────┐ │ ▼
45+
│ HOST │───┘ ┌─────────────┐ ┌─────────────┐
46+
│ Telegraf │ │ Grafana │◀────────────────│ Prometheus │
47+
└──────────┘ └─────────────┘ └─────────────┘
4848
4949
```
5050

‎charts/monitor/templates/_helpers.tmpl‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,3 +77,35 @@ env:
7777
- name: "REGISTRY_LOCATION"
7878
value: {{ .Values.global.registryLocation }}
7979
{{- end }}
80+
81+
{{/* Generate monitor telegraf envs */}}
82+
{{- define "telegraf.envs" }}
83+
env:
84+
- name: POD_NAMESPACE
85+
valueFrom:
86+
fieldRef:
87+
fieldPath: metadata.namespace
88+
- name: "AGENT_QUIET"
89+
value: "true"
90+
- name: "AGENT_BUFFER_LIMIT"
91+
value: "100000"
92+
- name: "ENABLE_KUBERNETES"
93+
value: "true"
94+
{{- if (.Values.telegraf.postgresqlConnection) }}
95+
- name: POSTGRESQL_CONNECTION
96+
value: {{ .Values.telegraf.postgresqlConnection }}
97+
{{- else if eq .Values.global.databaseLocation "on-cluster" }}
98+
- name: PG_USER
99+
valueFrom:
100+
secretKeyRef:
101+
name: database-creds
102+
key: user
103+
- name: PG_PASSWORD
104+
valueFrom:
105+
secretKeyRef:
106+
name: database-creds
107+
key: password
108+
- name: POSTGRESQL_CONNECTION
109+
value: "postgres://$(PG_USER):$(PG_PASSWORD)@drycc-database.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:5432/monitor"
110+
{{- end }}
111+
{{- end }}

‎charts/monitor/templates/monitor-telegraf-daemon.yaml‎

Lines changed: 12 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -29,76 +29,23 @@ spec:
2929
image: {{.Values.telegraf.imageRegistry}}/{{.Values.telegraf.imageOrg}}/python-dev:latest
3030
imagePullPolicy: {{.Values.imagePullPolicy}}
3131
args:
32-
- netcat
33-
- -v
34-
- -u
35-
- $(INFLUXDB_URLS_V2)
36-
env:
37-
{{- if eq .Values.global.influxdbLocation "off-cluster" }}
38-
- name: "INFLUXDB_URLS_V2"
39-
valueFrom:
40-
secretKeyRef:
41-
name: influxdb-creds
42-
key: url
43-
{{- else }}
44-
- name: "INFLUXDB_URLS_V2"
45-
value: "\"http://$(DRYCC_INFLUXDB_SERVICE_HOST):$(DRYCC_INFLUXDB_SERVICE_PORT)\""
46-
- name: "INFLUXDB_V2_INPUT_URLS"
47-
value: "\"http://$(DRYCC_INFLUXDB_SERVICE_HOST):$(DRYCC_INFLUXDB_SERVICE_PORT)/metrics\""
48-
- name: "ENABLE_INFLUXDB_V2_INPUT"
49-
value: "true"
50-
{{- end }}
32+
- netcat
33+
- -v
34+
- -u
35+
- $(POSTGRESQL_CONNECTION)
36+
{{- include "telegraf.envs" . | indent 8 }}
5137
containers:
5238
- name: drycc-monitor-telegraf
5339
image: {{.Values.telegraf.imageRegistry}}/{{.Values.telegraf.imageOrg}}/telegraf:{{.Values.telegraf.imageTag}}
5440
imagePullPolicy: {{.Values.telegraf.imagePullPolicy}}
55-
{{- if or (.Values.telegraf.limitsCpu) (.Values.telegraf.limitsMemory)}}
41+
{{- if or (.Values.telegraf.limitsCpu) (.Values.telegraf.limitsMemory)}}
5642
resources:
5743
limits:
58-
{{- if (.Values.telegraf.limitsCpu) }}
44+
{{- if (.Values.telegraf.limitsCpu) }}
5945
cpu: {{.Values.telegraf.limitsCpu}}
60-
{{- end}}
61-
{{- if (.Values.telegraf.limitsMemory) }}
46+
{{- end}}
47+
{{- if (.Values.telegraf.limitsMemory) }}
6248
memory: {{.Values.telegraf.limitsMemory}}
63-
{{- end}}
64-
{{- end}}
65-
env:
66-
- name: POD_NAMESPACE
67-
valueFrom:
68-
fieldRef:
69-
fieldPath: metadata.namespace
70-
{{- if eq .Values.global.influxdbLocation "off-cluster" }}
71-
- name: "INFLUXDB_URLS_V2"
72-
valueFrom:
73-
secretKeyRef:
74-
name: influxdb-creds
75-
key: url
76-
{{- else }}
77-
- name: "INFLUXDB_URLS_V2"
78-
value: "\"http://$(DRYCC_INFLUXDB_SERVICE_HOST):$(DRYCC_INFLUXDB_SERVICE_PORT)\""
79-
- name: "INFLUXDB_V2_INPUT_URLS"
80-
value: "\"http://$(DRYCC_INFLUXDB_SERVICE_HOST):$(DRYCC_INFLUXDB_SERVICE_PORT)/metrics\""
81-
- name: "ENABLE_INFLUXDB_V2_INPUT"
82-
value: "true"
83-
{{- end }}
84-
- name: "INFLUXDB_BUCKET"
85-
valueFrom:
86-
secretKeyRef:
87-
name: influxdb-creds
88-
key: bucket
89-
- name: "INFLUXDB_ORG"
90-
valueFrom:
91-
secretKeyRef:
92-
name: influxdb-creds
93-
key: org
94-
- name: "INFLUXDB_TOKEN"
95-
valueFrom:
96-
secretKeyRef:
97-
name: influxdb-creds
98-
key: token
99-
- name: "AGENT_QUIET"
100-
value: "true"
101-
- name: "AGENT_BUFFER_LIMIT"
102-
value: "100000"
103-
- name: "ENABLE_KUBERNETES"
104-
value: "true"
49+
{{- end}}
50+
{{- end}}
51+
{{- include "telegraf.envs" . | indent 8 }}

‎charts/monitor/values.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ telegraf:
4949
imageRegistry: "registry.drycc.cc"
5050
# limitsCpu: "100m"
5151
# limitsMemory: "50Mi"
52+
postgresqlConnection: ""
5253
nodeAffinityPreset:
5354
key: "drycc.cc/node"
5455
type: "soft"
@@ -77,7 +78,6 @@ global:
7778
# Valid values are:
7879
# - on-cluster: Run component within the Kubernetes cluster
7980
# - off-cluster: component is running outside of the cluster and credentials and connection information will be provided.
80-
influxdbLocation: "on-cluster"
8181
prometheusLocation: "on-cluster"
8282
grafanaLocation: "on-cluster"
8383
redisLocation: "on-cluster"

0 commit comments

Comments
 (0)