Skip to content
This repository was archived by the owner on Jun 25, 2025. It is now read-only.

Commit 7498fe5

Browse files
author
Jonathan Chauncey
committed
feat(model/message_handler): Add time to log message output
requires workflow-e2e#334 requires fluentd#61
1 parent d97e74c commit 7498fe5

8 files changed

Lines changed: 207 additions & 72 deletions

File tree

Makefile

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ clean: check-docker
7474
docker rmi $(IMAGE)
7575

7676
update-manifests:
77-
sed 's#\(image:\) .*#\1 $(IMAGE)#' manifests/deis-logger-rc.yaml > manifests/deis-logger-rc.tmp.yaml
77+
sed 's#\(image:\) .*#\1 $(IMAGE)#' manifests/deis-logger-deployment.yaml > manifests/deis-logger-deployment.tmp.yaml
7878

7979
test: test-style test-unit
8080

@@ -122,19 +122,19 @@ test-unit: start-test-redis
122122

123123
kube-install:
124124
kubectl create -f manifests/deis-logger-svc.yaml
125-
kubectl create -f manifests/deis-logger-rc.yaml
125+
kubectl create -f manifests/deis-logger-deployment.yaml
126126

127127
kube-delete:
128128
-kubectl delete -f manifests/deis-logger-svc.yaml
129-
-kubectl delete -f manifests/deis-logger-rc.tmp.yaml
129+
-kubectl delete -f manifests/deis-logger-deployment.tmp.yaml
130130

131131
kube-create: update-manifests
132132
kubectl create -f manifests/deis-logger-svc.yaml
133-
kubectl create -f manifests/deis-logger-rc.tmp.yaml
133+
kubectl create -f manifests/deis-logger-deployment.tmp.yaml
134134

135135
kube-replace: build push update-manifests
136-
kubectl replace --force -f manifests/deis-logger-rc.tmp.yaml
136+
kubectl replace --force -f manifests/deis-logger-deployment.tmp.yaml
137137

138138
kube-update: update-manifests
139-
kubectl delete -f manifests/deis-logger-rc.tmp.yaml
140-
kubectl create -f manifests/deis-logger-rc.tmp.yaml
139+
kubectl delete -f manifests/deis-logger-deployment.tmp.yaml
140+
kubectl create -f manifests/deis-logger-deployment.tmp.yaml

glide.lock

Lines changed: 15 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

glide.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,7 @@ import:
55
- package: github.com/kelseyhightower/envconfig
66
- package: github.com/nsqio/go-nsq
77
- package: gopkg.in/redis.v3
8+
- package: github.com/stretchr/testify
9+
version: ~1.1.4
10+
subpackages:
11+
- assert

log/message_handler.go

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,21 @@ import (
44
"encoding/json"
55
"fmt"
66
"regexp"
7+
"strings"
78

89
"github.com/deis/logger/storage"
910
)
1011

1112
const (
13+
podPattern = `(\w.*)-(\w.*)-(\w.*)-(\w.*)`
1214
controllerPattern = `^(INFO|WARN|DEBUG|ERROR)\s+(\[(\S+)\])+:(.*)`
1315
controllerContainerName = "deis-controller"
16+
timeFormat = "2006-01-02T15:04:05-07:00"
1417
)
1518

1619
var (
17-
regex = regexp.MustCompile(controllerPattern)
20+
controllerRegex = regexp.MustCompile(controllerPattern)
21+
podRegex = regexp.MustCompile(podPattern)
1822
)
1923

2024
func handle(rawMessage []byte, storageAdapter storage.Adapter) error {
@@ -23,7 +27,7 @@ func handle(rawMessage []byte, storageAdapter storage.Adapter) error {
2327
return err
2428
}
2529
if fromController(message) {
26-
storageAdapter.Write(getApplicationFromControllerMessage(message), message.Log)
30+
storageAdapter.Write(getApplicationFromControllerMessage(message), buildControllerLogMessage(message))
2731
} else {
2832
labels := message.Kubernetes.Labels
2933
storageAdapter.Write(labels["app"], buildApplicationLogMessage(message))
@@ -37,11 +41,23 @@ func fromController(message *Message) bool {
3741
}
3842

3943
func getApplicationFromControllerMessage(message *Message) string {
40-
return regex.FindStringSubmatch(message.Log)[3]
44+
return controllerRegex.FindStringSubmatch(message.Log)[3]
45+
}
46+
47+
func buildControllerLogMessage(message *Message) string {
48+
l := controllerRegex.FindStringSubmatch(message.Log)
49+
return fmt.Sprintf("%s deis[controller]: %s %s",
50+
message.Time.Format(timeFormat),
51+
l[1],
52+
strings.Trim(l[4], " "))
4153
}
4254

4355
func buildApplicationLogMessage(message *Message) string {
44-
body := message.Log
45-
podName := message.Kubernetes.PodName
46-
return fmt.Sprintf("%s -- %s", podName, body)
56+
return fmt.Sprintf("%s %s[%s.%s.%s]: %s",
57+
message.Time.Format(timeFormat),
58+
message.Kubernetes.Labels["app"],
59+
message.Kubernetes.Labels["type"],
60+
message.Kubernetes.Labels["version"],
61+
podRegex.FindStringSubmatch(message.Kubernetes.PodName)[4],
62+
message.Log)
4763
}

log/message_handler_test.go

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
package log
2+
3+
import (
4+
"encoding/json"
5+
"testing"
6+
7+
"github.com/deis/logger/storage"
8+
"github.com/stretchr/testify/assert"
9+
)
10+
11+
var (
12+
validControllerMessage = `{"log": "INFO [foo]: admin deployed 2fd9226", "time": "2016-10-18T20:29:38+00:00", "stream": "stderr", "docker": {"container_id": "containerId"}, "kubernetes": {"namespace_name": "foo", "pod_id": "podId", "pod_name": "podName", "container_name": "deis-controller", "labels": {"app": "foo",
13+
"heritage": "deis", "type": "web", "version": "v2"}, "host": "host"}}`
14+
15+
invalidControllerMessage = `{"log": "INFO: admin deployed 2fd9226", "stream": "stderr", "docker": {"container_id": "containerId"}, "kubernetes": {"namespace_name": "foo", "pod_id": "podId", "pod_name": "podName", "container_name": "deis-controller", "labels": {"app": "foo",
16+
"heritage": "deis", "type": "web", "version": "v2"}, "host": "host"}}`
17+
18+
validAppMessage = `{"log": "test message", "stream": "stderr", "time": "2016-10-18T20:29:38+00:00", "docker": {"container_id": "containerId"}, "kubernetes": {"namespace_name": "foo", "pod_id": "podId", "pod_name": "foo-web-845861952-nzf60", "container_name": "foo-web", "labels": {"app": "foo",
19+
"heritage": "deis", "type": "web", "version": "v2"}, "host": "host"}}`
20+
21+
badjson = `{"log":}`
22+
)
23+
24+
func TestValidControllerMessage(t *testing.T) {
25+
message := new(Message)
26+
err := json.Unmarshal([]byte(validControllerMessage), message)
27+
assert.NoError(t, err, "error occured parsing log message")
28+
assert.True(t, fromController(message), "json is not from controller")
29+
}
30+
31+
func TestInvalidControllerMessage(t *testing.T) {
32+
message := new(Message)
33+
err := json.Unmarshal([]byte(validAppMessage), message)
34+
assert.NoError(t, err, "error occured parsing log message")
35+
assert.False(t, fromController(message), "valid controller message")
36+
}
37+
38+
func TestGetApplicationFromValidControllerMessage(t *testing.T) {
39+
message := new(Message)
40+
err := json.Unmarshal([]byte(validControllerMessage), message)
41+
assert.NoError(t, err, "error occured parsing log message")
42+
expected := getApplicationFromControllerMessage(message)
43+
assert.Equal(t, expected, "foo", "failed to retrieve app from message")
44+
}
45+
46+
func TestBuildControllerLogMessageFromValidMessage(t *testing.T) {
47+
message := new(Message)
48+
err := json.Unmarshal([]byte(validControllerMessage), message)
49+
assert.NoError(t, err, "error occured parsing log message")
50+
expected := buildControllerLogMessage(message)
51+
assert.Equal(t, expected,
52+
"2016-10-18T20:29:38+00:00 deis[controller]: INFO admin deployed 2fd9226",
53+
"failed to build controller log")
54+
}
55+
56+
func TestBuildApplicationLogMessageFromValidMessage(t *testing.T) {
57+
message := new(Message)
58+
err := json.Unmarshal([]byte(validAppMessage), message)
59+
assert.NoError(t, err, "error occured parsing log message")
60+
expected := buildApplicationLogMessage(message)
61+
assert.Equal(t, expected,
62+
"2016-10-18T20:29:38+00:00 foo[web.v2.nzf60]: test message",
63+
"failed to build application log")
64+
}
65+
66+
func TestHandleValidAppMessage(t *testing.T) {
67+
a, err := storage.NewRingBufferAdapter(1)
68+
assert.NoError(t, err, "error creating ring buffer")
69+
err = handle([]byte(validAppMessage), a)
70+
assert.NoError(t, err, "error occured storing log message")
71+
expected, _ := a.Read("foo", 1)
72+
assert.Equal(t, expected[0],
73+
"2016-10-18T20:29:38+00:00 foo[web.v2.nzf60]: test message",
74+
"failed to aquire application log message")
75+
}
76+
77+
func TestHandleValidControllerMessage(t *testing.T) {
78+
a, err := storage.NewRingBufferAdapter(1)
79+
assert.NoError(t, err, "error creating ring buffer")
80+
err = handle([]byte(validControllerMessage), a)
81+
assert.NoError(t, err, "error occured storing log message")
82+
expected, _ := a.Read("foo", 1)
83+
assert.Equal(t, expected[0],
84+
"2016-10-18T20:29:38+00:00 deis[controller]: INFO admin deployed 2fd9226",
85+
"failed to aquire controller log message")
86+
}
87+
88+
func TestHandleInvalidAppMessage(t *testing.T) {
89+
a, err := storage.NewRingBufferAdapter(1)
90+
assert.NoError(t, err, "error creating ring buffer")
91+
err = handle([]byte(validAppMessage), a)
92+
assert.NoError(t, err, "error occured storing log message")
93+
expected, _ := a.Read("foo", 1)
94+
assert.Equal(t, expected[0],
95+
"2016-10-18T20:29:38+00:00 foo[web.v2.nzf60]: test message",
96+
"failed to aquire application log message")
97+
}
98+
99+
func TestHandleInvalidControllerMessage(t *testing.T) {
100+
a, err := storage.NewRingBufferAdapter(1)
101+
assert.NoError(t, err, "error creating ring buffer")
102+
err = handle([]byte(badjson), a)
103+
assert.Error(t, err, "no error occured parsing json")
104+
}

log/model.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package log
22

3+
import "time"
4+
35
// Example log message JSON:
46
//
57
// {"log"=>"2016/05/31 01:34:43 10.164.1.1 GET / - 5074209722772702441\n", "stream"=>"stderr",
@@ -15,6 +17,7 @@ type Message struct {
1517
Stream string `json:"stream"`
1618
Kubernetes Kubernetes `json:"kubernetes"`
1719
Docker Docker `json:"docker"`
20+
Time time.Time `json:"time"`
1821
}
1922

2023
// Kubernetes specific log message fields
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
apiVersion: extensions/v1beta1
2+
kind: Deployment
3+
metadata:
4+
name: deis-logger
5+
namespace: deis
6+
labels:
7+
heritage: deis
8+
annotations:
9+
helm-keep: "true"
10+
component.deis.io/version: canary
11+
spec:
12+
replicas: 1
13+
strategy:
14+
rollingUpdate:
15+
maxSurge: 1
16+
maxUnavailable: 0
17+
type: RollingUpdate
18+
selector:
19+
matchLabels:
20+
app: deis-logger
21+
template:
22+
metadata:
23+
labels:
24+
app: deis-logger
25+
spec:
26+
containers:
27+
- name: deis-logger
28+
image: quay.io/deis/logger:canary
29+
imagePullPolicy: Always
30+
env:
31+
- name: STORAGE_ADAPTER
32+
value: redis
33+
- name: DEIS_LOGGER_REDIS_PASSWORD
34+
valueFrom:
35+
secretKeyRef:
36+
name: logger-redis-creds
37+
key: password
38+
ports:
39+
- containerPort: 8088
40+
name: http
41+
livenessProbe:
42+
httpGet:
43+
path: /healthz
44+
port: 8088
45+
initialDelaySeconds: 1
46+
timeoutSeconds: 1
47+
readinessProbe:
48+
httpGet:
49+
path: /healthz
50+
port: 8088
51+
initialDelaySeconds: 1
52+
timeoutSeconds: 1

tests/logger_test.go

Lines changed: 0 additions & 56 deletions
This file was deleted.

0 commit comments

Comments
 (0)