feat(deis-output): Custom fluentd plugin for sending data to deis components

Jonathan Chauncey · Jonathan Chauncey · commit cb4cc44189f2 · 2016-06-15T10:46:30.000-06:00
All data from this plugin is sent to an nsq instance where consumers
will pull that data off and push/store it in the correct location.
The topics where data is stored is configurable in both instances by
setting NSQ_LOG_TOPIC and NSQ_METRIC_TOPIC. Default is logs and metrics
respectively.
diff --git a/.gitignore b/.gitignore
@@ -1 +1,3 @@
 manifests/*.tmp.yaml
+rootfs/opt/fluentd/deis-output/pkg
+rootfs/*.gem
diff --git a/Makefile b/Makefile
@@ -18,7 +18,7 @@ docker-build:
 kube-delete:
 	-kubectl delete -f manifests/deis-logger-fluentd-daemon.tmp.yaml
 
-kube-create: update-manifests
+kube-install: update-manifests
 	kubectl create -f manifests/deis-logger-fluentd-daemon.tmp.yaml
 
 kube-update: update-manifests
@@ -27,3 +27,6 @@ kube-update: update-manifests
 
 update-manifests:
 	sed 's#\(image:\) .*#\1 $(IMAGE)#' manifests/deis-logger-fluentd-daemon.yaml > manifests/deis-logger-fluentd-daemon.tmp.yaml
+
+test: docker-build
+	docker run ${IMAGE} /bin/bash -c "cd /opt/fluentd/deis-output && rake test"
diff --git a/README.md b/README.md
@@ -27,3 +27,14 @@ This plugin allows `fluentd` to send data to a remote syslog endpoint like [pape
 * `SYSLOG_PORT_2=52232`
 
 You can also set `SYSLOG_HOST` and `SYSLOG_PORT`..
+
+### Deis Output
+Deis output is a custom fluentd plugin that was written to forward data directly to deis components while filtering out data that we did not care about. We have 2 pieces of information we care about currently.
+
+1) Logs from applications that are written to stdout within the container and the controller logs that represent actions against those applications. These logs are sent to an internal messaging system ([NSQ](http://nsq.io)) on a configurable topic. The logger component then reads those messages and stores the data in an ring buffer.
+
+2) Metric data from the nginx based router. We take the log and parse out `request_time`, `response_time`, and `bytes_sent`. Each one of these metrics makes up a series that we will ultimately send to our InfluxDB system. Attached to each series is the host the data came from (where router is running) and the status code for that request.
+
+The topics these messages are put on are configurable via environment variables.
+* `NSQ_LOG_TOPIC`
+* `NSQ_METRIC_TOPIC`
diff --git a/manifests/deis-logger-fluentd-daemon.yaml b/manifests/deis-logger-fluentd-daemon.yaml
@@ -31,19 +31,12 @@ spec:
           mountPath: /var/lib/docker/containers
           readOnly: true
         env:
-        - name: SYSLOG_HOST_1
-          value: $(DEIS_LOGGER_SERVICE_HOST)
-        - name: SYSLOG_PORT_1
-          value: $(DEIS_LOGGER_SERVICE_PORT_TRANSPORT)
-        - name: SYSLOG_HOST_2
-          value: $(DEIS_MONITOR_STDOUT_SERVICE_HOST)
-        - name: SYSLOG_PORT_2
-          value: $(DEIS_MONITOR_STDOUT_SERVICE_PORT_TRANSPORT)
+        - name: DEBUG
+          value: "true"
       volumes:
       - name: varlog
         hostPath:
           path: /var/log
       - name: varlibdockercontainers
         hostPath:
           path: /var/lib/docker/containers
-
diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile
@@ -1,30 +1,34 @@
-FROM quay.io/deis/base:0.2.0
+FROM quay.io/deis/base:0.3.0
 
 COPY . /
 
 RUN apt-get update \
-	&& apt-get install -y \
-		g++ \
-		gcc \
-		make \
-		ruby \
-		ruby-dev \
-	&& echo 'gem: --no-document' >> /etc/gemrc \
-	&& export FLUENTD_VERSION=0.14.0.pre.1 \
-	&& gem install --no-document fluentd -v $FLUENTD_VERSION \
-	&& gem install --no-document fluent-plugin-kubernetes_metadata_filter \
-	&& gem install --no-document fluent-plugin-elasticsearch \
-	&& gem install --no-document fluent-plugin-remote_syslog -v 0.3.2 \
-	&& apt-get remove -y --auto-remove --purge \
-		g++ \
-		gcc \
-		make \
-		ruby-dev \
-	&& apt-get clean \
-	&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man /usr/share/doc
+  && apt-get install -y \
+    g++ \
+    gcc \
+    make \
+    ruby \
+    ruby-dev \
+  && export FLUENTD_VERSION=0.14.0 \
+  && gem install --no-document fluentd -v $FLUENTD_VERSION \
+  && gem install bundler \
+  && bundle install --gemfile=/opt/fluentd/deis-output/Gemfile \
+  && rake --rakefile=/opt/fluentd/deis-output/Rakefile build \
+  && fluent-gem install --no-document fluent-plugin-kubernetes_metadata_filter \
+  && fluent-gem install --no-document fluent-plugin-elasticsearch \
+  && fluent-gem install --no-document fluent-plugin-remote_syslog -v 0.3.2 \
+  && fluent-gem install --no-document influxdb -v 0.3.2 \
+  && fluent-gem install --no-document nsq-ruby \
+  && fluent-gem install --local /opt/fluentd/deis-output/pkg/fluent-plugin-deis_output-0.1.0.gem \
+  && apt-get remove -y --auto-remove --purge \
+    g++ \
+    gcc \
+    make \
+    ruby-dev \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man /usr/share/doc
 
-# Re-copy this monkey patch because a previous step would have overwritten it
-COPY var/lib/gems/2.3.0/gems/fluent-plugin-remote_syslog-0.3.2/lib/fluent/plugin/out_remote_syslog.rb /var/lib/gems/2.3.0/gems/fluent-plugin-remote_syslog-0.3.2/lib/fluent/plugin/out_remote_syslog.rb
+COPY /var /var
 
 CMD ["/opt/fluentd/sbin/boot"]
 
diff --git a/rootfs/opt/fluentd/deis-output/.ruby-version b/rootfs/opt/fluentd/deis-output/.ruby-version
@@ -0,0 +1 @@
+2.2.2
diff --git a/rootfs/opt/fluentd/deis-output/Gemfile b/rootfs/opt/fluentd/deis-output/Gemfile
@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec
diff --git a/rootfs/opt/fluentd/deis-output/Gemfile.lock b/rootfs/opt/fluentd/deis-output/Gemfile.lock
@@ -0,0 +1,68 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-deis_output (0.1.0)
+      fluent-mixin-config-placeholders
+      fluent-mixin-plaintextformatter
+      fluent-mixin-rewrite-tag-name
+      fluentd
+      influxdb (~> 0.3)
+      nsq-ruby
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    cool.io (1.4.4)
+    fluent-mixin-config-placeholders (0.4.0)
+      fluentd
+      uuidtools (>= 2.1.5)
+    fluent-mixin-plaintextformatter (0.2.6)
+      fluentd
+      ltsv
+    fluent-mixin-rewrite-tag-name (0.1.0)
+      fluentd
+    fluentd (0.14.0)
+      cool.io (>= 1.4.3, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      json (>= 1.4.3)
+      msgpack (>= 0.7.0)
+      serverengine (>= 1.6.4)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.1.7)
+      tzinfo (>= 1.0.0)
+      tzinfo-data (>= 1.0.0)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.6.0)
+    influxdb (0.3.2)
+      json
+    json (1.8.3)
+    ltsv (0.1.0)
+    msgpack (0.7.6)
+    nsq-ruby (1.6.0)
+    power_assert (0.3.0)
+    rake (10.5.0)
+    serverengine (1.6.4)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.1.8)
+    test-unit (3.1.9)
+      power_assert
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2016.4)
+      tzinfo (>= 1.0.0)
+    uuidtools (2.1.5)
+    yajl-ruby (1.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.3)
+  fluent-plugin-deis_output!
+  rake (~> 10.0)
+  test-unit (~> 3.1.7)
+
+BUNDLED WITH
+   1.12.3
diff --git a/rootfs/opt/fluentd/deis-output/LICENSE b/rootfs/opt/fluentd/deis-output/LICENSE
@@ -0,0 +1,13 @@
+Copyright Engine Yard, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/rootfs/opt/fluentd/deis-output/README.md b/rootfs/opt/fluentd/deis-output/README.md
@@ -0,0 +1,21 @@
+# Deis Fluentd plugin
+Deis (pronounced DAY-iss) Workflow is an open source Platform as a Service (PaaS) that adds a developer-friendly layer to any [Kubernetes](http://kubernetes.io) cluster, making it easy to deploy and manage applications on your own servers.
+
+For more information about the Deis Workflow, please visit the main project page at https://github.com/deis/workflow.
+
+## About
+[Fluentd](http://www.fluentd.org/) is an integral part of the [Deis Workflow](https://github.com/deis/workflow) stack. It is responsible for consuming all of the log data produced by the Kuberenetes cluster. This includes the kubernetes components, Workflow components, and user deployed applications. This plugin takes that log data and will forward it to the appropriate Workflow components.
+
+## License
+
+© Engine Yard, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may
+not use this file except in compliance with the License. You may obtain
+a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0>
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/rootfs/opt/fluentd/deis-output/Rakefile b/rootfs/opt/fluentd/deis-output/Rakefile
@@ -0,0 +1,11 @@
+require 'bundler'
+gemdir = File.realpath(File.join(File.dirname(__FILE__), "."))
+Bundler::GemHelper.install_tasks :dir => gemdir, :name => 'fluent-plugin-deis_output'
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/**/test_*.rb']
+  test.verbose = true
+end
diff --git a/rootfs/opt/fluentd/deis-output/fluent-plugin-deis_output.gemspec b/rootfs/opt/fluentd/deis-output/fluent-plugin-deis_output.gemspec
@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-deis_output"
+  gem.version       = "0.1.0"
+  gem.authors       = ["engineering@deis"]
+  gem.email         = ["engineering@deis.com"]
+  gem.description   = %q{Fluentd output plugin for sending data to Deis Components }
+  gem.summary       = %q{Fluentd output plugin for sending data to Deis Components}
+  gem.homepage      = "https://github.com/deis/fluentd"
+  gem.license       = "ASL2"
+
+  gem.files         = Dir.glob("{lib}/**/*") + %w(LICENSE README.md)
+  gem.executables   = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.has_rdoc      = false
+
+  gem.required_ruby_version = '>= 2.0.0'
+
+  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "fluent-mixin-plaintextformatter"
+  gem.add_runtime_dependency "fluent-mixin-config-placeholders"
+  gem.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
+  gem.add_runtime_dependency "influxdb", '~> 0.3'
+  gem.add_runtime_dependency "nsq-ruby"
+
+  gem.add_development_dependency "bundler", "~> 1.3"
+  gem.add_development_dependency "rake", "~> 10.0"
+  gem.add_development_dependency "test-unit", "~> 3.1.7"
+end
diff --git a/rootfs/opt/fluentd/deis-output/lib/fluent/mixin/deis.rb b/rootfs/opt/fluentd/deis-output/lib/fluent/mixin/deis.rb
@@ -0,0 +1,118 @@
+require 'json'
+require 'nsq'
+require 'influxdb'
+
+module Fluent
+  module Mixin
+    module Deis
+      LOGGER_URL = "http://#{ENV['DEIS_LOGGER_SERVICE_HOST']}:#{ENV['DEIS_LOGGER_SERVICE_PORT_HTTP']}/logs"
+      INFLUX_HOST = "#{ENV['DEIS_MONITOR_INFLUXAPI_SERVICE_HOST']}"
+      INFLUX_PORT = "#{ENV['DEIS_MONITOR_INFLUXAPI_SERVICE_PORT_TRANSPORT']}"
+      INFLUX_DATABASE = ENV['INFLUX_DATABASE'] || "kubernetes"
+      NSQ_URL = "#{ENV['DEIS_NSQD_SERVICE_HOST']}:#{ENV['DEIS_NSQD_SERVICE_PORT_TRANSPORT']}"
+
+      def kubernetes?(message)
+        return message["kubernetes"] != nil
+      end
+
+      def from_controller?(message)
+        if from_container?(message, "deis-controller")
+          return message["log"] =~ /^(INFO|WARN|DEBUG|ERROR)\s+(\[(\S+)\])+:(.*)/
+        end
+        return false
+      end
+
+      def from_router?(message)
+        return from_container?(message, "deis-router")
+      end
+
+      def from_container?(message, regex)
+        if kubernetes? message
+          return true if Regexp.new(regex).match(message["kubernetes"]["container_name"]) != nil
+        end
+        return false
+      end
+
+      def deis_deployed_app?(message)
+        if kubernetes? message
+          labels = message["kubernetes"]["labels"]
+          return true if message["kubernetes"]["namespace_name"] != "deis" && labels["heritage"] == "deis" && labels["app"] != nil
+        end
+        return false
+      end
+
+      def push(producer, value)
+        begin
+          if value.kind_of? Hash
+            producer.write(value.to_json)
+          else
+            producer.write(value)
+          end
+        rescue Exception => e
+          puts "Error:#{map.message}"
+        end
+      end
+
+      def build_series(message)
+        metric = parse_router_log(message["log"], message["kubernetes"]["host"])
+        if metric
+          tags = { app: metric["app"], status_code: metric["status_code"], host: metric["host"] }
+          data = [
+            {
+              series: 'deis_router_request_time_ms',
+              values: { value: metric["request_time"] },
+              tags: tags
+            },
+            {
+              series: 'deis_router_response_time_ms',
+              values: { value: metric["response_time"] },
+              tags: tags
+            },
+            {
+              series: 'deis_router_bytes_sent',
+              values: { value: metric["bytes_sent"] },
+              tags: tags
+            },
+          ]
+          return data
+        end
+        return nil
+      end
+
+      # {"log"=>"[2016-05-31T16:56:12+00:00] - foo - 10.164.1.1 - - - 200 - \"GET / HTTP/1.0\" - 211 - \"-\" - \"ApacheBench/2.3\" - \"~^foo\\x5C.(?<domain>.+)$\" - 10.167.243.4:80 - foo.devart.io - 0.002 - 0.046\n"}
+      # request_time - request processing time in seconds with a milliseconds resolution; time elapsed between the first bytes were read from the client and the log write after the last bytes were sent to the client
+      # response_time - keeps time spent on receiving the response from the upstream server; the time is kept in seconds with millisecond resolution.
+      def parse_router_log(message, host)
+        begin
+          split_message = message.split(" - ")
+          return nil if split_message.length < 14
+          metric = {}
+          metric["app"] = split_message[1].strip
+          metric["status_code"] = split_message[4].strip
+          metric["bytes_sent"] = split_message[6].strip.to_f
+          metric["response_time"] = split_message[12].strip.to_f
+          metric["request_time"] = split_message[13].strip.to_f
+          metric["host"] = host
+          return metric
+        rescue Exception => e
+          puts "Error:#{e.message}"
+          return nil
+        end
+      end
+
+      def debug?
+        ENV["DEBUG"] == "true"
+      end
+
+      def get_nsq_producer(topic)
+        begin
+          puts "Creating nsq producer (#{NSQ_URL}) for topic:#{topic}"
+          return Nsq::Producer.new(nsqd: NSQ_URL, topic: topic)
+        rescue Exception => e
+          puts "Error:#{e.message}"
+          return nil
+        end
+      end
+    end
+  end
+end
diff --git a/rootfs/opt/fluentd/deis-output/lib/fluent/plugin/out_deis.rb b/rootfs/opt/fluentd/deis-output/lib/fluent/plugin/out_deis.rb
diff --git a/rootfs/opt/fluentd/deis-output/test/test_out_deis.rb b/rootfs/opt/fluentd/deis-output/test/test_out_deis.rb
diff --git a/rootfs/opt/fluentd/sbin/boot b/rootfs/opt/fluentd/sbin/boot

Original file line number	Diff line number	Diff line change
`@@ -1 +1,3 @@`
`1`	`1`	`manifests/*.tmp.yaml`
	`2`	`+rootfs/opt/fluentd/deis-output/pkg`
	`3`	`+rootfs/*.gem`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+source 'https://rubygems.org'`
	`2`	`+`
	`3`	`+gemspec`