Skip to content

Commit 9ad2c0d

Browse files
committed
initial commit. fully operational
0 parents  commit 9ad2c0d

10 files changed

Lines changed: 847 additions & 0 deletions

File tree

Dockerfile

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
FROM flynn/busybox
2+
MAINTAINER Jeff Lindsay <progrium@gmail.com>
3+
4+
ADD ./build/logspout /bin/logspout
5+
6+
ENV ROUTESPATH /mnt/routes
7+
VOLUME /mnt/routes
8+
9+
EXPOSE 8000
10+
11+
ENTRYPOINT ["/bin/logspout"]
12+
CMD []

LICENSE

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Copyright (C) 2014 Jeff Lindsay
2+
3+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
4+
5+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
6+
7+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Makefile

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
build/container: build/logspout Dockerfile
2+
docker build --no-cache -t logspout .
3+
touch build/container
4+
5+
build/logspout: *.go
6+
go build -o build/logspout
7+
8+
.PHONY: clean
9+
clean:
10+
rm -rf build

README.md

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
# logspout
2+
3+
A log router and HTTP interface for Docker container log streams, made to run inside Docker. Besides the routes you make, it's a stateless log appliance. It's not meant for managing log files or looking at history, just a means to get your logs out to live somewhere else, where they belong.
4+
5+
## Getting and running
6+
7+
Logspout is a (very small) Docker container, so you can just pull it from the index:
8+
9+
$ docker pull progrium/logspout
10+
11+
When running logspout, it exposes port 8000 and needs two mounts. The first is the Docker Unix socket. The second is a directory to persist routes. We mount both with `-v`:
12+
13+
$ docker run -d -P \
14+
-v=/var/run/docker.sock:/var/run/docker.sock \
15+
-v=/var/lib/logspout:/mnt/routes \
16+
progrium/logspout
17+
18+
Both need to be mounted in these specific paths inside the container, but where you keep the routes on the host could be anywhere. It could also be a regular Docker volume. If you don't mount a volume at `/mnt/routes`, it will only store routes in memory.
19+
20+
You can optionally pass an argument to install a catch-all route in the form `<type>://<addr>`. For example, to route all logs via syslog to `192.168.1.111:514`, run like this:
21+
22+
$ docker run -d -P \
23+
-v=/var/run/docker.sock:/var/run/docker.sock \
24+
-v=/var/lib/logspout:/mnt/routes \
25+
progrium/logspout syslog://192.168.1.111:514
26+
27+
## HTTP API
28+
29+
### Streaming Endpoints
30+
31+
You can use these chunked transfer streaming endpoints for quick debugging with `curl` or for setting up easy TCP subscriptions to log sources. They also support WebSocket upgrades.
32+
33+
GET /logs
34+
GET /logs/filter:<container-name-substring>
35+
GET /logs/id:<container-id>
36+
GET /logs/name:<container-name>
37+
38+
You can select specific log types from a source using a comma-delimited list in the query param `types`. Right now the only types are `stdout` and `stderr`, but when Docker properly takes over each container's syslog socket (or however they end up doing it), other types will be possible.
39+
40+
If you include a request `Accept: application/json` header, the output will be JSON objects including the name and ID of the container and the log type. Note that when upgrading to WebSocket, it will always use JSON.
41+
42+
Since `/logs` and `/logs/filter:<string>` endpoints can return logs from multiple source, they will by default return color-coded loglines prefixed with the name of the container. You can turn off the color escape codes with query param `colors=off` or the alternative is to stream the data in JSON format, which won't use colors or prefixes.
43+
44+
45+
### Routing Resource
46+
47+
Routes let you configure logspout to hand-off logs to another system. Right now the only supported target type is via UDP `syslog`, but hey that's pretty much everything.
48+
49+
#### Creating a route
50+
51+
POST /routes
52+
53+
Takes a JSON object like this:
54+
55+
{
56+
"source": {
57+
"filter": "_db"
58+
"types": ["stdout"]
59+
},
60+
"target": {
61+
"type": "syslog",
62+
"addr": "logaggregator.service.consul"
63+
"append_tag": ".db"
64+
}
65+
}
66+
67+
The `source` field should be an object with `filter`, `name`, or `id` fields. You can specify specific log types with the `types` field to collect only `stdout` or `stderr`. If you don't specify `types`, it will route all types.
68+
69+
To route all logs of all types on all containers, don't specify a `source`.
70+
71+
The `append_tag` field of `target` is optional and specific to `syslog`. It lets you append to the tag of syslog packets for this route. By default the tag is `<container-name>`, so an `append_tag` value of `.app` would make the tag `<container-name>.app`.
72+
73+
And yes, you can just specify an IP and port for `addr`, but you can also specify a name that resolves via DNS to one or more SRV records. That means this works great with [Consul](http://www.consul.io/) for service discovery.
74+
75+
#### Listing routes
76+
77+
GET /routes
78+
79+
Returns a JSON list of current routes:
80+
81+
[
82+
{
83+
"id": "3631c027fb1b",
84+
"source": {
85+
"name": "mycontainer"
86+
},
87+
"target": {
88+
"type": "syslog",
89+
"addr": "192.168.1.111:514"
90+
}
91+
}
92+
]
93+
94+
#### Viewing a route
95+
96+
GET /routes/<id>
97+
98+
Returns a JSON route object:
99+
100+
{
101+
"id": "3631c027fb1b",
102+
"source": {
103+
"id": "a9efd0aeb470"
104+
"types": ["stderr"]
105+
},
106+
"target": {
107+
"type": "syslog",
108+
"addr": "192.168.1.111:514"
109+
}
110+
}
111+
112+
#### Deleting a route
113+
114+
DELETE /routes/<id>
115+
116+
## Sponsor
117+
118+
This project was made possible by [DigitalOcean](http://digitalocean.com).
119+
120+
## License
121+
122+
BSD

SPONSORS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
DigitalOcean http://digitalocean.com

attacher.go

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
package main
2+
3+
import (
4+
"bufio"
5+
"io"
6+
"log"
7+
"strings"
8+
"sync"
9+
10+
"github.com/fsouza/go-dockerclient"
11+
)
12+
13+
type AttachManager struct {
14+
sync.Mutex
15+
attached map[string]*LogPump
16+
channels map[chan *AttachEvent]struct{}
17+
client *docker.Client
18+
}
19+
20+
func NewAttachManager(client *docker.Client) *AttachManager {
21+
m := &AttachManager{
22+
attached: make(map[string]*LogPump),
23+
channels: make(map[chan *AttachEvent]struct{}),
24+
client: client,
25+
}
26+
containers, err := client.ListContainers(docker.ListContainersOptions{})
27+
assert(err, "attacher")
28+
for _, listing := range containers {
29+
m.attach(listing.ID[:12])
30+
}
31+
go func() {
32+
events := make(chan *docker.APIEvents)
33+
assert(client.AddEventListener(events), "attacher")
34+
for msg := range events {
35+
debug("event:", msg.ID[:12], msg.Status)
36+
if msg.Status == "create" {
37+
go m.attach(msg.ID[:12])
38+
}
39+
}
40+
log.Fatal("ruh roh") // todo: loop?
41+
}()
42+
return m
43+
}
44+
45+
func (m *AttachManager) attach(id string) {
46+
container, err := m.client.InspectContainer(id)
47+
assert(err, "attacher")
48+
name := container.Name[1:]
49+
success := make(chan struct{})
50+
failure := make(chan error)
51+
outrd, outwr := io.Pipe()
52+
errrd, errwr := io.Pipe()
53+
go func() {
54+
err := m.client.AttachToContainer(docker.AttachToContainerOptions{
55+
Container: id,
56+
OutputStream: outwr,
57+
ErrorStream: errwr,
58+
Stdin: false,
59+
Stdout: true,
60+
Stderr: true,
61+
Stream: true,
62+
Success: success,
63+
})
64+
outwr.Close()
65+
errwr.Close()
66+
debug("attach:", id, "finished")
67+
if err != nil {
68+
close(success)
69+
failure <- err
70+
}
71+
m.send(&AttachEvent{Type: "detach", ID: id, Name: name})
72+
m.Lock()
73+
delete(m.attached, id)
74+
m.Unlock()
75+
}()
76+
_, ok := <-success
77+
if ok {
78+
m.Lock()
79+
m.attached[id] = NewLogPump(outrd, errrd, id, name)
80+
m.Unlock()
81+
success <- struct{}{}
82+
m.send(&AttachEvent{ID: id, Name: name, Type: "attach"})
83+
debug("attach:", id, "success")
84+
return
85+
}
86+
debug("attach:", id, "failure:", <-failure)
87+
}
88+
89+
func (m *AttachManager) send(event *AttachEvent) {
90+
m.Lock()
91+
defer m.Unlock()
92+
for ch, _ := range m.channels {
93+
// TODO: log err after timeout and continue
94+
ch <- event
95+
}
96+
}
97+
98+
func (m *AttachManager) addListener(ch chan *AttachEvent) {
99+
m.Lock()
100+
defer m.Unlock()
101+
m.channels[ch] = struct{}{}
102+
go func() {
103+
for id, pump := range m.attached {
104+
ch <- &AttachEvent{ID: id, Name: pump.Name, Type: "attach"}
105+
}
106+
}()
107+
}
108+
109+
func (m *AttachManager) removeListener(ch chan *AttachEvent) {
110+
m.Lock()
111+
defer m.Unlock()
112+
delete(m.channels, ch)
113+
}
114+
115+
func (m *AttachManager) Get(id string) *LogPump {
116+
m.Lock()
117+
defer m.Unlock()
118+
return m.attached[id]
119+
}
120+
121+
func (m *AttachManager) Listen(source *Source, logstream chan *Log, closer <-chan bool) {
122+
if source == nil {
123+
source = new(Source)
124+
}
125+
events := make(chan *AttachEvent)
126+
m.addListener(events)
127+
defer m.removeListener(events)
128+
for {
129+
select {
130+
case event := <-events:
131+
if event.Type == "attach" && (source.All() ||
132+
(source.ID != "" && strings.HasPrefix(event.ID, source.ID)) ||
133+
(source.Name != "" && event.Name == source.Name) ||
134+
(source.Filter != "" && strings.Contains(event.Name, source.Filter))) {
135+
pump := m.Get(event.ID)
136+
pump.AddListener(logstream)
137+
defer func() {
138+
if pump != nil {
139+
pump.RemoveListener(logstream)
140+
}
141+
}()
142+
} else if source.ID != "" && event.Type == "detach" &&
143+
strings.HasPrefix(event.ID, source.ID) {
144+
return
145+
}
146+
case <-closer:
147+
return
148+
}
149+
}
150+
}
151+
152+
type LogPump struct {
153+
sync.Mutex
154+
ID string
155+
Name string
156+
channels map[chan *Log]struct{}
157+
}
158+
159+
func NewLogPump(stdout, stderr io.Reader, id, name string) *LogPump {
160+
obj := &LogPump{
161+
ID: id,
162+
Name: name,
163+
channels: make(map[chan *Log]struct{}),
164+
}
165+
pump := func(typ string, source io.Reader) {
166+
buf := bufio.NewReader(source)
167+
for {
168+
data, err := buf.ReadBytes('\n')
169+
if err != nil {
170+
if err != io.EOF {
171+
debug("pump:", id, typ+":", err)
172+
}
173+
return
174+
}
175+
obj.send(&Log{
176+
Data: strings.TrimSuffix(string(data), "\n"),
177+
ID: id,
178+
Name: name,
179+
Type: typ,
180+
})
181+
}
182+
}
183+
go pump("stdout", stdout)
184+
go pump("stderr", stderr)
185+
return obj
186+
}
187+
188+
func (o *LogPump) send(log *Log) {
189+
o.Lock()
190+
defer o.Unlock()
191+
for ch, _ := range o.channels {
192+
// TODO: log err after timeout and continue
193+
ch <- log
194+
}
195+
}
196+
197+
func (o *LogPump) AddListener(ch chan *Log) {
198+
o.Lock()
199+
defer o.Unlock()
200+
o.channels[ch] = struct{}{}
201+
}
202+
203+
func (o *LogPump) RemoveListener(ch chan *Log) {
204+
o.Lock()
205+
defer o.Unlock()
206+
delete(o.channels, ch)
207+
}

0 commit comments

Comments
 (0)