diff --git a/Dockerfile b/Dockerfile index 0b62a8c..e9ab496 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ ARG CODENAME FROM registry.drycc.cc/drycc/base:${CODENAME} -ENV DRYCC_UID=1001 \ +ARG DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/data \ - NODE_EXPORTER_VERSION="1.9.1" \ - KUBE_STATE_METRICS="2.15.0" \ - VICTORIAMETRICS_VERSION="1.116.0" + NODE_EXPORTER_VERSION="1.10.2" \ + KUBE_STATE_METRICS="2.17.0" \ + VICTORIAMETRICS_VERSION="1.129.1" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} \ diff --git a/charts/victoriametrics/templates/victoriametrics/networkpolicy.yaml b/charts/victoriametrics/templates/victoriametrics/networkpolicy.yaml index 4c281db..6b394b9 100644 --- a/charts/victoriametrics/templates/victoriametrics/networkpolicy.yaml +++ b/charts/victoriametrics/templates/victoriametrics/networkpolicy.yaml @@ -1,7 +1,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: drycc-victoriametrics-networkpolicy + name: drycc-victoriametrics spec: podSelector: matchLabels: @@ -13,4 +13,13 @@ spec: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: {{.Release.Namespace}} - podSelector: {} + podSelector: + matchLabels: + app: drycc-controller + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{.Release.Namespace}} + podSelector: + matchLabels: + app: drycc-victoriametrics diff --git a/charts/victoriametrics/templates/victoriametrics/vmagent/vmagent-configmap.yaml b/charts/victoriametrics/templates/victoriametrics/vmagent/vmagent-configmap.yaml index dfc1e99..ed99185 100644 --- a/charts/victoriametrics/templates/victoriametrics/vmagent/vmagent-configmap.yaml +++ b/charts/victoriametrics/templates/victoriametrics/vmagent/vmagent-configmap.yaml @@ -11,41 +11,107 @@ data: {{- (tpl .Values.vmagent.prometheus $) | nindent 4 }} {{- else }} global: - ## How frequently to scrape targets by default - ## scrape_interval: 1m - ## How long until a scrape request times out - ## scrape_timeout: 10s scrape_configs: - job_name: vmagent static_configs: - targets: ["localhost:8429"] + - job_name: "controller-nodes" + scheme: http + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - target_label: __param_scheme + replacement: https + - target_label: __param_host + replacement: kubernetes.default.svc + - target_label: __param_port + replacement: 443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __param_path + replacement: /api/v1/nodes/$1/proxy/metrics + - target_label: __address__ + replacement: drycc-controller-metric:8000 + - target_label: __metrics_path__ + replacement: /v2/metrics + - job_name: "controller-nodes-cadvisor" + scheme: http + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - target_label: __param_scheme + replacement: https + - target_label: __param_host + replacement: kubernetes.default.svc + - target_label: __param_port + replacement: 443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __param_path + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + - target_label: __address__ + replacement: drycc-controller-metric:8000 + - target_label: __metrics_path__ + replacement: /v2/metrics + - job_name: "controller-pods" + scheme: http + honor_labels: true + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + source_labels: [__meta_kubernetes_pod_container_init] + regex: true + - action: keep_if_equal + source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_container_port_number] + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - target_label: __param_scheme + replacement: http + - source_labels: [__address__] + regex: ([^:]+)(?::\d+)? + target_label: __param_host + replacement: $1 + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_container_port_number] + regex: (\d+);.*|.*;(\d+) + target_label: __param_port + replacement: ${1}${2} + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + regex: (.+) + target_label: __param_path + replacement: $1 + - source_labels: [__param_path] + regex: ^$ + replacement: /metrics + target_label: __param_path + - target_label: __address__ + replacement: drycc-controller-metric:8000 + - target_label: __metrics_path__ + replacement: /v2/metrics + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_pod_container_name] + target_label: container + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_node_name] + action: replace + target_label: node - job_name: "kubernetes-apiservers" kubernetes_sd_configs: - role: endpoints - # Default to scraping over https. If required, just disable this or change to - # `http`. scheme: https - # This TLS & bearer token file config is used to connect to the actual scrape - # endpoints for cluster components. This is separate to discovery auth - # configuration because discovery & scraping are two separate concerns in - # Prometheus. The discovery auth config is automatic if Prometheus runs inside - # the cluster. Otherwise, more config options have to be provided within the - # . tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - # If your node certificates are self-signed or use a different CA to the - # master CA, then you need to disable certificate verification. Note that - # certificate verification is an integral part of a secure infrastructure - # so this should only be disabled in a controlled environment. You can - # enable certificate verification by commenting the line below. - # insecure_skip_verify: true bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - # Keep only the default/kubernetes service endpoints for the https port. This - # will add targets for each API server which Kubernetes adds an endpoint to - # the default/kubernetes service. relabel_configs: - source_labels: [ @@ -56,23 +122,9 @@ data: action: keep regex: default;kubernetes;https - job_name: "kubernetes-nodes" - # Default to scraping over https. If required, just disable this or change to - # `http`. scheme: https - # This TLS & bearer token file config is used to connect to the actual scrape - # endpoints for cluster components. This is separate to discovery auth - # configuration because discovery & scraping are two separate concerns in - # Prometheus. The discovery auth config is automatic if Prometheus runs inside - # the cluster. Otherwise, more config options have to be provided within the - # . tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - # If your node certificates are self-signed or use a different CA to the - # master CA, then you need to disable certificate verification. Note that - # certificate verification is an integral part of a secure infrastructure - # so this should only be disabled in a controlled environment. You can - # enable certificate verification by commenting the line below. - # insecure_skip_verify: true bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: @@ -87,32 +139,13 @@ data: target_label: __metrics_path__ replacement: /api/v1/nodes/$1/proxy/metrics - job_name: "kubernetes-nodes-cadvisor" - # Default to scraping over https. If required, just disable this or change to - # `http`. scheme: https - # This TLS & bearer token file config is used to connect to the actual scrape - # endpoints for cluster components. This is separate to discovery auth - # configuration because discovery & scraping are two separate concerns in - # Prometheus. The discovery auth config is automatic if Prometheus runs inside - # the cluster. Otherwise, more config options have to be provided within the - # . tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - # If your node certificates are self-signed or use a different CA to the - # master CA, then disable certificate verification below. Note that - # certificate verification is an integral part of a secure infrastructure - # so this should only be disabled in a controlled environment. You can - # disable certificate verification by uncommenting the line below. - # insecure_skip_verify: true bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: - role: node - # This configuration will work only on kubelet 1.7.3+ - # As the scrape endpoints for cAdvisor have changed - # if you are using older version you need to change the replacement to - # replacement: /api/v1/nodes/$1:4194/proxy/metrics - # more info here https://github.com/coreos/prometheus-operator/issues/633 relabel_configs: - action: labelmap regex: __meta_kubernetes_node_label_(.+) @@ -162,7 +195,7 @@ data: ] separator: ; regex: (.+);(.+) - replacement: $1.$2.svc.cluster.local + replacement: $1.$2.svc target_label: __address__ - source_labels: [ @@ -172,9 +205,10 @@ data: ] separator: ; regex: (.+);(.+);(\d+) - replacement: $1.$2.svc.cluster.local:$3 + replacement: $1.$2.svc:$3 target_label: __address__ - job_name: "kubernetes-service-endpoints" + honor_labels: true kubernetes_sd_configs: - role: endpointslices relabel_configs: @@ -192,11 +226,14 @@ data: separator: ; regex: "true;true" action: drop - - source_labels: - [__meta_kubernetes_service_annotation_prometheus_io_scheme] - action: replace - target_label: __scheme__ + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] regex: (https?) + replacement: $1 + target_label: __scheme__ + - source_labels: [__scheme__] + regex: ^$ + replacement: http + target_label: __scheme__ - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] action: replace @@ -228,6 +265,7 @@ data: action: replace target_label: node - job_name: "kubernetes-service-endpoints-slow" + honor_labels: true scrape_interval: 5m scrape_timeout: 30s kubernetes_sd_configs: @@ -242,11 +280,14 @@ data: [__meta_kubernetes_service_annotation_prometheus_io_scrape_slow] action: keep regex: true - - source_labels: - [__meta_kubernetes_service_annotation_prometheus_io_scheme] - action: replace - target_label: __scheme__ + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] regex: (https?) + replacement: $1 + target_label: __scheme__ + - source_labels: [__scheme__] + regex: ^$ + replacement: http + target_label: __scheme__ - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] action: replace @@ -278,6 +319,7 @@ data: action: replace target_label: node - job_name: "kubernetes-services" + honor_labels: true metrics_path: /probe params: module: [http_2xx] @@ -301,6 +343,7 @@ data: - source_labels: [__meta_kubernetes_service_name] target_label: service - job_name: "kubernetes-pods" + honor_labels: true kubernetes_sd_configs: - role: pod relabel_configs: @@ -312,6 +355,14 @@ data: - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] action: keep regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + regex: (https?) + replacement: $1 + target_label: __scheme__ + - source_labels: [__scheme__] + regex: ^$ + replacement: http + target_label: __scheme__ - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] action: replace target_label: __metrics_path__ diff --git a/charts/victoriametrics/templates/victoriametrics/vmagent/vmagent-statefulset.yaml b/charts/victoriametrics/templates/victoriametrics/vmagent/vmagent-statefulset.yaml index e7aac0c..3880916 100644 --- a/charts/victoriametrics/templates/victoriametrics/vmagent/vmagent-statefulset.yaml +++ b/charts/victoriametrics/templates/victoriametrics/vmagent/vmagent-statefulset.yaml @@ -23,15 +23,19 @@ spec: podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.vmagent.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.vmagent.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }} nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.vmagent.nodeAffinityPreset.type "key" .Values.vmagent.nodeAffinityPreset.key "values" .Values.vmagent.nodeAffinityPreset.values ) | nindent 10 }} serviceAccount: drycc-victoriametrics-vmagent + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsUser: 1001 initContainers: - - name: drycc-victoriametrics-vmstorage-init + - name: drycc-victoriametrics-vmagent-init image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest imagePullPolicy: {{.Values.imagePullPolicy}} args: - netcat - -v - - -a - - {{ printf "drycc-victoriametrics-vminsert.%s.svc.%s:8480" $.Release.Namespace $.Values.global.clusterDomain }} + - -u + - http://drycc-controller-api,http://drycc-victoriametrics-vminsert:8480 containers: - name: drycc-victoriametrics-vmagent image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/victoriametrics:{{.Values.imageTag}} @@ -42,7 +46,7 @@ spec: {{- else }} args: - vmagent - - --remoteWrite.url={{ printf "http://drycc-victoriametrics-vminsert.%s.svc.%s:8480/insert/0/prometheus/api/v1/write" $.Release.Namespace $.Values.global.clusterDomain }} + - --remoteWrite.url=http://drycc-victoriametrics-vminsert:8480/insert/multitenant/prometheus/api/v1/write - --remoteWrite.tmpDataPath=/data - --httpListenAddr=:8429 - --promscrape.config=/opt/drycc/victoriametrics/config/prometheus.yaml diff --git a/charts/victoriametrics/templates/victoriametrics/vminsert/vminsert-deployment.yaml b/charts/victoriametrics/templates/victoriametrics/vminsert/vminsert-deployment.yaml index 4897844..884d0b5 100644 --- a/charts/victoriametrics/templates/victoriametrics/vminsert/vminsert-deployment.yaml +++ b/charts/victoriametrics/templates/victoriametrics/vminsert/vminsert-deployment.yaml @@ -1,7 +1,7 @@ {{- $replicas := .Values.vmstorage.replicas | int -}} {{- $endpoints := list -}} {{- range $i := until $replicas -}} - {{- $endpoint := printf "drycc-victoriametrics-vmstorage-%d.drycc-victoriametrics-vmstorage.%s.svc.%s:8400" $i $.Release.Namespace $.Values.global.clusterDomain -}} + {{- $endpoint := printf "drycc-victoriametrics-vmstorage-%d.drycc-victoriametrics-vmstorage:8400" $i -}} {{- $endpoints = append $endpoints $endpoint -}} {{- end -}} {{- $storageNodes := join "," $endpoints -}} diff --git a/charts/victoriametrics/templates/victoriametrics/vmselect/vmselect-deployment.yaml b/charts/victoriametrics/templates/victoriametrics/vmselect/vmselect-deployment.yaml index 7929443..9ca114f 100644 --- a/charts/victoriametrics/templates/victoriametrics/vmselect/vmselect-deployment.yaml +++ b/charts/victoriametrics/templates/victoriametrics/vmselect/vmselect-deployment.yaml @@ -1,7 +1,7 @@ {{- $replicas := .Values.vmstorage.replicas | int -}} {{- $endpoints := list -}} {{- range $i := until $replicas -}} - {{- $endpoint := printf "drycc-victoriametrics-vmstorage-%d.drycc-victoriametrics-vmstorage.%s.svc.%s:8401" $i $.Release.Namespace $.Values.global.clusterDomain -}} + {{- $endpoint := printf "drycc-victoriametrics-vmstorage-%d.drycc-victoriametrics-vmstorage:8401" $i -}} {{- $endpoints = append $endpoints $endpoint -}} {{- end -}} {{- $storageNodes := join "," $endpoints -}} diff --git a/charts/victoriametrics/templates/victoriametrics/vmstorage/vmstorage-statefulset.yaml b/charts/victoriametrics/templates/victoriametrics/vmstorage/vmstorage-statefulset.yaml index 1d555be..b27a19b 100644 --- a/charts/victoriametrics/templates/victoriametrics/vmstorage/vmstorage-statefulset.yaml +++ b/charts/victoriametrics/templates/victoriametrics/vmstorage/vmstorage-statefulset.yaml @@ -23,6 +23,10 @@ spec: podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.vmstorage.podAffinityPreset.type "component" "" "extraMatchLabels" .Values.vmstorage.podAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }} podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.vmstorage.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.vmstorage.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }} nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.vmstorage.nodeAffinityPreset.type "key" .Values.vmstorage.nodeAffinityPreset.key "values" .Values.vmstorage.nodeAffinityPreset.values ) | nindent 10 }} + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsUser: 1001 containers: - name: drycc-victoriametrics-vmstorage image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/victoriametrics:{{.Values.imageTag}} diff --git a/charts/victoriametrics/values.yaml b/charts/victoriametrics/values.yaml index 8638be7..18ef170 100644 --- a/charts/victoriametrics/values.yaml +++ b/charts/victoriametrics/values.yaml @@ -216,13 +216,5 @@ kubeStateMetrics: - storageclasses - validatingwebhookconfigurations - volumeattachments - # - verticalpodautoscalers # not a default resource, see also: https://github.com/kubernetes/kube-state-metrics#enabling-verticalpodautoscalers - -global: - # Admin email, used for each component to send email to administrator - email: "drycc@drycc.cc" - # A domain name consists of one or more parts. - # Periods (.) are used to separate these parts. - # Each part must be 1 to 63 characters in length and can contain lowercase letters, digits, and hyphens (-). - # It must start and end with a lowercase letter or digit. - clusterDomain: "cluster.local" + metricAnnotationsAllowList: + - persistentvolumeclaims=[*]