Merge pull request #66 from jchauncey/use-prometheus

feat(telegraf): Add the ability to poll kubernetes prometheus endpoints

Author: Jonathan Chauncey

telegraf/manifests/deis-monitor-telegraf-daemon.yaml

@@ -16,30 +16,32 @@ spec:
           image: quay.io/deis/telegraf:v2-beta
           imagePullPolicy: Always
           env:
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
           - name: "INFLUXDB_URLS"
-            value: http://$(DEIS_MONITOR_INFLUXDB_SERVICE_HOST):$(DEIS_MONITOR_INFLUXDB_PORT_8086_TCP_PORT)
-          - name: "KUBERNETES_DOCKER_ENDPOINT"
-            value: "unix:///var/run/docker.sock"
-          # - name: "HOST_PROC"
-          #   value: "/rootfs/proc"
-          # - name: "HOST_SYS"
-          #   value: "/rootfs/sys"
+            value: http://$(DEIS_MONITOR_INFLUXDB_SERVICE_HOST):$(DEIS_MONITOR_INFLUXDB_SERVICE_PORT_TRANSPORT)
+          - name: "HOST_PROC"
+            value: "/rootfs/proc"
+          - name: "HOST_SYS"
+            value: "/rootfs/sys"
           volumeMounts:
           - mountPath: /var/run/docker.sock
             name: docker-socket
-          # - name: sysro
-          #   mountPath: /rootfs/sys
-          #   readOnly: true
-          # - name: procro
-          #   mountPath: /rootfs/proc
-          #   readOnly: true
+          - name: sysro
+            mountPath: /rootfs/sys
+            readOnly: true
+          - name: procro
+            mountPath: /rootfs/proc
+            readOnly: true
       volumes:
       - name: docker-socket
         hostPath:
           path: /var/run/docker.sock
-      # - name: sysro
-      #   hostPath:
-      #     path: /sys
-      # - name: procro
-      #   hostPath:
-      #     path: /proc
+      - name: sysro
+        hostPath:
+          path: /sys
+      - name: procro
+        hostPath:
+          path: /proc

telegraf/rootfs/Dockerfile

@@ -3,7 +3,7 @@ MAINTAINER Jonathan Chauncey "<jchauncey@deis.com>"
 
 ENV TELEGRAF_VERSION "0.10.0_linux_amd64"
 
-RUN apk --update add bash
+RUN apk --update add bash curl
 
 # The download links for the zip of telegraf are broken.
 # So for now Im building this locally and copying it into the build context.

telegraf/rootfs/config.toml.tpl

@@ -11,7 +11,7 @@
   flush_jitter = "{{ default "0s" .AGENT_FLUSH_JITTER }}"
   debug = {{ default "false" .AGENT_DEBUG }}
   quiet = {{ default "false" .AGENT_QUIET }}
-  {{ if .AGENT_HOSTNAME }} hostname = {{ .AGENT_HOSTNAME }} {{ end }}
+  {{ if .AGENT_HOSTNAME }}hostname = "{{ .AGENT_HOSTNAME }}"{{ end }}
 
 # Set output configuration
 {{ if .AMON_INSTANCE }}
@@ -135,12 +135,6 @@
   container_names = [{{ if .DOCKER_CONTAINER_NAMES }}{{ .DOCKER_CONTAINER_NAMES }}{{ end }}]
 {{ end }}
 
-{{ if .KUBERNETES_SERVICE_HOST}}
-[[inputs.kubernetes]]
-  docker_endpoint = "{{ .KUBERNETES_DOCKER_ENDPOINT }}"
-  container_names = [{{ if .KUBERNETES_CONTAINER_NAMES }}{{ .KUBERNETES_CONTAINER_NAMES }}{{ end }}]
-{{ end }}
-
 {{ if .ELASTIC_SEARCH_SERVERS }}
 [[inputs.elasticsearch]]
   servers = [{{ .ELASTIC_SEARCH_SERVERS }}]
@@ -200,6 +194,12 @@
   {{ if .POSTGRESQL_DATABASES }} databases = [{{ .POSTGRESQL_DATABASES }}]  {{ end }}
 {{ end }}
 
+{{ if .PROMETHEUS_URLS }}
+[[inputs.prometheus]]
+  urls = [{{ .PROMETHEUS_URLS }}]
+  bearer_token = "{{ .PROMETHEUS_BEARER_TOKEN }}"
+{{ end }}
+
 {{ if .RABBITMQ_URL }}
 [[inputs.rabbitmq]]
   url = "{{ .RABBITMQ_URL }}"

telegraf/rootfs/start-telegraf

@@ -1,4 +1,13 @@
 #!/bin/bash
+export PROMETHEUS_BEARER_TOKEN=/var/run/secrets/kubernetes.io/serviceaccount/token
+export TOKEN=$(cat $PROMETHEUS_BEARER_TOKEN)
+export POD_API_URL=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/$POD_NAMESPACE/pods/$HOSTNAME
+export NODE_NAME=$(curl -s $POD_API_URL --header "Authorization: Bearer $TOKEN" --insecure | grep nodeName | cut -c 18- | tr -d '"')
+export AGENT_HOSTNAME=$NODE_NAME
+echo "Setting Agent Hostname to: $AGENT_HOSTNAME"
+export PROMETHEUS_URLS="\"https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/proxy/nodes/$NODE_NAME/metrics\", \"https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/metrics\""
+echo "Setting PROMETHEUS_URLS to: $PROMETHEUS_URLS"
+
 echo "Building config.toml!"
 ./envtpl -in config.toml.tpl >> config.toml
 echo "Finished building toml..."