feat(config): Allows for building custom plugins

This plugin reverts the revert we made last release.
fixes #43

Author: Jonathan Chauncey

README.md

@@ -46,3 +46,16 @@ Deis output is a custom fluentd plugin that was written to forward data directly
 The topics these messages are put on are configurable via environment variables.
 * `NSQ_LOG_TOPIC`
 * `NSQ_METRIC_TOPIC`
+
+### Custom Plugins
+If you need something beyond the plugins that come pre-installed in the image, it is possible to set some environment variables to install and configure custom plugins as well.
+
+To install a custom plugin, simply set a FLUENTD_PLUGIN_# environment variable. For multiple plugins simply increment the trailing number.
+`FLUENTD_PLUGIN_1=some-fluentd-plugin`
+
+To configure your custom plugins, use either the CUSTOM_STORE_# or CUSTOM_FILTER_# environment variables
+* `CUSTOM_STORE_1="configuration text"`
+* `CUSTOM_FILTER_1="configuration text"`
+
+If you need the build tools available for installing your plugin, this can be enabled with another environment variable
+`INSTALL_BUILD_TOOLS="true"`

rootfs/opt/fluentd/sbin/boot

@@ -1,6 +1,16 @@
 #!/bin/bash
-FLUENTD_CONF="/opt/fluentd/conf/fluentd.conf"
+FLUENTD_CONF=${FLUENTD_CONF:-"/opt/fluentd/conf/fluentd.conf"}
+INSTALL_BUILD_TOOLS=${INSTALL_BUILD_TOOLS:-"false"}
+BUILD_TOOLS='g++ gcc make ruby-dev'
 
+if [ $INSTALL_BUILD_TOOLS == "true" ]
+then
+  echo "Installing Build tools!"
+  apt-get update
+  apt-get install -y ruby $BUILD_TOOLS
+fi
+
+source /opt/fluentd/sbin/plugins
 source /opt/fluentd/sbin/sources
 source /opt/fluentd/sbin/filters/filters
 
@@ -15,4 +25,12 @@ cat << EOF >> $FLUENTD_CONF
 </match>
 EOF
 
+if [ $INSTALL_BUILD_TOOLS == "true" ]
+then
+  echo "Removing Build tools!"
+  apt-get remove -y --auto-remove --purge $BUILD_TOOLS
+  apt-get clean
+  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man /usr/share/doc
+fi
+
 exec fluentd -c $FLUENTD_CONF

rootfs/opt/fluentd/sbin/filters/custom_filters

@@ -0,0 +1,12 @@
+FILTERS=$(env | grep -E '^(CUSTOM_FILTER_\d*)' | sed 's/=.*//')
+
+for filter in $FILTERS
+do
+  echo "Configuring $filter"
+  conf=$(eval "echo \"\$$filter\"")
+  if [ -n "$conf" ]
+  then
+    echo -e "\n# $filter" >> $FLUENTD_CONF
+    echo "$conf" >> $FLUENTD_CONF
+  fi
+done

rootfs/opt/fluentd/sbin/filters/filters

@@ -1,2 +1,3 @@
 #!/bin/bash
 source /opt/fluentd/sbin/filters/kubernetes
+source /opt/fluentd/sbin/filters/custom_filters

rootfs/opt/fluentd/sbin/plugins

@@ -0,0 +1,10 @@
+PLUGINS=$(env | grep -E '^(FLUENTD_PLUGIN_\d*)' | sed 's/=.*//')
+for plugin_var in $PLUGINS
+do
+  plugin=$(eval "echo \$$plugin_var")
+  if [ -n "$plugin" ]
+  then
+    echo "Installing fluentd plugin $plugin."
+    fluent-gem install --no-document $plugin
+  fi
+done

rootfs/opt/fluentd/sbin/stores/custom_stores

@@ -0,0 +1,12 @@
+STORES=$(env | grep -E '^(CUSTOM_STORE_\d*)' | sed 's/=.*//')
+
+for store in $STORES
+do
+  echo "Configuring $store"
+  conf=$(eval "echo \"\$$store\"")
+  if [ -n "$conf" ]
+  then
+    echo -e "\n# $store" >> $FLUENTD_CONF
+    echo "$conf" >> $FLUENTD_CONF
+  fi
+done

rootfs/opt/fluentd/sbin/stores/stores

@@ -4,3 +4,4 @@ source /opt/fluentd/sbin/stores/deis
 source /opt/fluentd/sbin/stores/elastic_search
 source /opt/fluentd/sbin/stores/syslog
 source /opt/fluentd/sbin/stores/sumologic
+source /opt/fluentd/sbin/stores/custom_stores