feat(logger): add new syslog daemon based on https://github.com/ziutek/syslog

Author: Gabriel Monroy

logger/Dockerfile

@@ -1,23 +1,25 @@
 FROM deis/base:latest
 MAINTAINER Gabriel Monroy <gabriel@opdemand.com>
 
-# install rsyslog from rsyslog.com
-RUN echo "deb http://ppa.launchpad.net/adiscon/v7-stable/ubuntu precise main" > /etc/apt/sources.list.d/rsyslog.list
-RUN echo "deb-src http://ppa.launchpad.net/adiscon/v7-stable/ubuntu precise main" >> /etc/apt/sources.list.d/rsyslog.list
-RUN gpg --recv-keys --keyserver keyserver.ubuntu.com AEF0CF8E
-RUN gpg --export --armor AEF0CF8E | sudo apt-key add -
-RUN apt-get update
-RUN apt-get install -yq --force-yes rsyslog
+# install go runtime
+RUN wget -O /tmp/go1.2.1.linux-amd64.tar.gz -q https://go.googlecode.com/files/go1.2.1.linux-amd64.tar.gz
+RUN tar -C /usr/local -xzvf /tmp/go1.2.1.linux-amd64.tar.gz
 
-# create /var/log/deis for holding logs (access via bind mount)
-RUN mkdir -p /var/log/deis
-
-# install latest etcdctl including no-sync options
-RUN wget -q https://s3-us-west-2.amazonaws.com/deis/etcdctl.no-sync -O /usr/local/bin/etcdctl
-RUN chmod +x /usr/local/bin/etcdctl
+# prepare go environment
+RUN mkdir -p /go
+ENV GOPATH /go
+ENV PATH /usr/local/bin:/usr/bin:/bin:/sbin:/usr/local/go/bin
 
 # add the current build context to /app
 ADD . /app
+ADD syslog /go/src/syslog
+ADD syslogd /go/src/syslogd
+
+# compile the binary
+RUN cd /go/src/syslogd && go install -v syslogd
+
+# create /var/log/deis for holding logs (access via bind mount)
+RUN mkdir -p /var/log/deis
 
 # prepare execution environment
 WORKDIR /app

logger/bin/boot

@@ -15,31 +15,11 @@ export ETCD="$HOST:$ETCD_PORT"
 export ETCD_PATH=${ETCD_PATH:-/deis/logs}
 export ETCD_TTL=${ETCD_TTL:-10}
 
-# wait for etcd to be available
-until etcdctl --no-sync -C $ETCD ls >/dev/null; do
-	echo "waiting for etcd at $ETCD..."
-	sleep $(($ETCD_TTL/2))  # sleep for half the TTL
-done
-
-# wait until etcd has discarded potentially stale values
-sleep $(($ETCD_TTL+1))
-
-# seed initial service configuration if necessary
-if ! etcdctl --no-sync -C $ETCD ls $ETCD_PATH >/dev/null 2>&1; then
-	etcdctl --no-sync -C $ETCD mkdir $ETCD_PATH >/dev/null 2>&1 || true
-fi
-
-# wait for confd to run once and install initial templates
-until confd -onetime -node $ETCD -config-file /app/confd.toml; do
-	echo "logger: waiting for confd to write initial templates..."
-	sleep $(($ETCD_TTL/2))  # sleep for half the TTL
-done
-
-# cleanup old pidfile's
-test -e /var/run/rsyslog.pid && rm -f /var/run/rsyslog.pid
+# fix perms on log directory
+chmod 755 /var/log/deis
 
 # spawn the service in the background
-/usr/sbin/rsyslogd -n &
+/go/bin/syslogd &
 SERVICE_PID=$!
 
 # smart shutdown on SIGINT and SIGTERM
@@ -49,10 +29,6 @@ function on_exit() {
 }
 trap on_exit INT TERM
 
-# spawn confd in the background to update services based on etcd changes
-confd -node $ETCD -config-file /app/confd.toml &
-CONFD_PID=$!
-
 echo deis-logger running...
 
 # publish the service to etcd using the injected PORT

logger/conf.d/default.toml

@@ -0,0 +1,24 @@
+Copyright (c) 2012, Michal Derkacz
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. The name of the author may not be used to endorse or promote products
+   derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

logger/conf.d/deis.conf.toml

@@ -0,0 +1,94 @@
+package syslog
+
+import (
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+// FileHandler implements Handler interface in the way to save messages into a
+// text file. It properly handles logrotate HUP signal (closes a file and tries
+// to open/create new one).
+type FileHandler struct {
+	bh       *BaseHandler
+	filename string
+	f        *os.File
+	l        Logger
+}
+
+// NewFileHandler accepts all arguments expected by NewBaseHandler plus
+// filename which is the path to the log file.
+func NewFileHandler(filename string, qlen int, filter func(*Message) bool,
+	ft bool) *FileHandler {
+
+	h := &FileHandler{
+		bh:       NewBaseHandler(qlen, filter, ft),
+		filename: filename,
+		l:        log.New(os.Stderr, "", log.LstdFlags),
+	}
+	go h.mainLoop()
+	return h
+}
+
+// SetLogger changes an internal logger used to log I/O errors. By default I/O
+// errors are written to os.Stderr using log.Logger.
+func (h *FileHandler) SetLogger(l Logger) {
+	h.l = l
+}
+
+func (h *FileHandler) mainLoop() {
+	defer h.bh.End()
+
+	mq := h.bh.Queue()
+	sq := make(chan os.Signal, 1)
+	signal.Notify(sq, syscall.SIGHUP)
+
+	for {
+		select {
+		case <-sq: // SIGHUP probably from logrotate
+			h.checkErr(h.f.Close())
+			h.f = nil
+		case m, ok := <-mq: // message to save
+			if !ok {
+				if h.f != nil {
+					h.checkErr(h.f.Close())
+				}
+				return
+			}
+			h.saveMessage(m)
+		}
+	}
+}
+
+func (h *FileHandler) saveMessage(m *Message) {
+	var err error
+	if h.f == nil {
+		h.f, err = os.OpenFile(
+			h.filename,
+			os.O_WRONLY|os.O_APPEND|os.O_CREATE,
+			0620,
+		)
+		if h.checkErr(err) {
+			return
+		}
+	}
+	_, err = h.f.WriteString(m.String() + "\n")
+	h.checkErr(err)
+}
+
+func (h *FileHandler) checkErr(err error) bool {
+	if err == nil {
+		return false
+	}
+	if h.l == nil {
+		log.Print(h.filename, ": ", err)
+	} else {
+		h.l.Print(h.filename, ": ", err)
+	}
+	return true
+}
+
+func (h *FileHandler) Handle(m *Message) *Message {
+	return h.bh.Handle(m)
+}

logger/conf.d/rsyslog.conf.toml

@@ -0,0 +1,81 @@
+package syslog
+
+// Handler handles syslog messages
+type Handler interface {
+	// Handle should return Message (mayby modified) for future processing by
+	// other handlers or return nil. If Handle is called with nil message it
+	// should complete all remaining work and properly shutdown before return.
+	Handle(*Message) *Message
+}
+
+// BaseHandler is desigend for simplify the creation of real handlers. It
+// implements Handler interface using nonblocking queuing of messages and
+// simple message filtering.
+type BaseHandler struct {
+	queue  chan *Message
+	end    chan struct{}
+	filter func(*Message) bool
+	ft     bool
+}
+
+// NewBaseHandler creates BaseHandler using specified filter. If filter is nil
+// or if it returns true messages are passed to BaseHandler internal queue
+// (of qlen length). If filter returns false or ft is true messages are returned
+// to server for future processing by other handlers.
+func NewBaseHandler(qlen int, filter func(*Message) bool, ft bool) *BaseHandler {
+	return &BaseHandler{
+		queue:  make(chan *Message, qlen),
+		end:    make(chan struct{}),
+		filter: filter,
+		ft:     ft,
+	}
+}
+
+// Handle inserts m in an internal queue. It immediately returns even if
+// queue is full. If m == nil it closes queue and waits for End method call
+// before return.
+func (h *BaseHandler) Handle(m *Message) *Message {
+	if m == nil {
+		close(h.queue) // signal that ther is no more messages for processing
+		<-h.end        // wait for handler shutdown
+		return nil
+	}
+	if h.filter != nil && !h.filter(m) {
+		// m doesn't match the filter
+		return m
+	}
+	// Try queue m
+	select {
+	case h.queue <- m:
+	default:
+	}
+	if h.ft {
+		return m
+	}
+	return nil
+}
+
+// Get returns first message from internal queue. It waits for message if queue
+// is empty. It returns nil if there is no more messages to process and handler
+// should shutdown.
+func (h *BaseHandler) Get() *Message {
+	m, ok := <-h.queue
+	if ok {
+		return m
+	}
+	return nil
+}
+
+// Queue returns BaseHandler internal queue as read-only channel. You can use
+// it directly, especially if your handler need to select from multiple channels
+// or have to work without blocking. You need to check if this channel is closed by
+// sender and properly shutdown in this case.
+func (h *BaseHandler) Queue() <-chan *Message {
+	return h.queue
+}
+
+// End signals the server that handler properly shutdown. You need to call End
+// only if Get has returned nil before.
+func (h *BaseHandler) End() {
+	close(h.end)
+}

logger/confd.toml

@@ -0,0 +1,15 @@
+package syslog
+
+// Logger is an interface for package internal (non fatal) logging
+type Logger interface {
+	Print(...interface{})
+	Printf(format string, v ...interface{})
+	Println(...interface{})
+}
+
+// FatalLogger is an interface for logging package internal fatal errors
+type FatalLogger interface {
+	Fatal(...interface{})
+	Fatalf(format string, v ...interface{})
+	Fatalln(...interface{})
+}