Skip to content
This repository was archived by the owner on Jun 25, 2025. It is now read-only.

Commit 424523c

Browse files
krancourkrancour
committed
feat(storage): Add redis storage adapter
1 parent 8a65360 commit 424523c

8 files changed

Lines changed: 297 additions & 12 deletions

File tree

‎Makefile‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@ REPO_PATH = github.com/deis/logger
1515
# and other build options
1616
DEV_ENV_IMAGE := quay.io/deis/go-dev:0.13.0
1717
DEV_ENV_WORK_DIR := /go/src/${REPO_PATH}
18-
DEV_ENV_CMD := docker run --rm -v ${CURDIR}:${DEV_ENV_WORK_DIR} -w ${DEV_ENV_WORK_DIR} ${DEV_ENV_IMAGE}
19-
DEV_ENV_CMD_INT := docker run -it --rm -v ${CURDIR}:${DEV_ENV_WORK_DIR} -w ${DEV_ENV_WORK_DIR} ${DEV_ENV_IMAGE}
18+
DEV_ENV_OPTS := --rm -v ${CURDIR}:${DEV_ENV_WORK_DIR} -w ${DEV_ENV_WORK_DIR}
19+
DEV_ENV_CMD := docker run ${DEV_ENV_OPTS} ${DEV_ENV_IMAGE}
20+
DEV_ENV_CMD_INT := docker run -it ${DEV_ENV_OPTS} ${DEV_ENV_IMAGE}
2021
LDFLAGS := "-s -X main.version=${VERSION}"
2122

2223
BINARY_DEST_DIR = rootfs/opt/logger/sbin
@@ -29,6 +30,8 @@ IMAGE_PREFIX ?= deis
2930

3031
include versioning.mk
3132

33+
REDIS_CONTAINER_NAME := test-redis-${VERSION}
34+
3235
SHELL_SCRIPTS = $(wildcard _scripts/*.sh)
3336

3437
check-docker:
@@ -75,8 +78,15 @@ update-manifests:
7578

7679
test: test-style test-unit
7780

78-
test-cover:
79-
${DEV_ENV_CMD} test-cover.sh
81+
test-cover: start-test-redis
82+
docker run ${DEV_ENV_OPTS} \
83+
-it \
84+
--link ${REDIS_CONTAINER_NAME}:TEST_REDIS \
85+
${DEV_ENV_IMAGE} bash -c 'DEIS_LOGGER_REDIS_SERVICE_HOST=$$TEST_REDIS_PORT_6379_TCP_ADDR \
86+
DEIS_LOGGER_REDIS_SERVICE_PORT=$$TEST_REDIS_PORT_6379_TCP_PORT \
87+
test-cover.sh' \
88+
|| (make stop-test-redis && false)
89+
make stop-test-redis
8090

8191
test-style: check-docker
8292
${DEV_ENV_CMD} make style-check
@@ -89,8 +99,22 @@ style-check:
8999
$(GOLINT) ./...
90100
shellcheck $(SHELL_SCRIPTS)
91101

92-
test-unit:
93-
${DEV_ENV_CMD} $(GOTEST) $$(glide nv)
102+
start-test-redis:
103+
docker run --name ${REDIS_CONTAINER_NAME} -d redis:latest
104+
105+
stop-test-redis:
106+
docker kill ${REDIS_CONTAINER_NAME}
107+
docker rm ${REDIS_CONTAINER_NAME}
108+
109+
test-unit: start-test-redis
110+
docker run ${DEV_ENV_OPTS} \
111+
-it \
112+
--link ${REDIS_CONTAINER_NAME}:TEST_REDIS \
113+
${DEV_ENV_IMAGE} bash -c 'DEIS_LOGGER_REDIS_SERVICE_HOST=$$TEST_REDIS_PORT_6379_TCP_ADDR \
114+
DEIS_LOGGER_REDIS_SERVICE_PORT=$$TEST_REDIS_PORT_6379_TCP_PORT \
115+
$(GOTEST) $$(glide nv)' \
116+
|| (make stop-test-redis && false)
117+
make stop-test-redis
94118

95119
kube-install:
96120
kubectl create -f manifests/deis-logger-svc.yaml

‎glide.lock‎

Lines changed: 11 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎glide.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@ import:
44
- package: github.com/gorilla/context
55
- package: github.com/kelseyhightower/envconfig
66
- package: github.com/nsqio/go-nsq
7+
- package: gopkg.in/redis.v3

‎storage/factory.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package storage
33
import (
44
"fmt"
55
"github.com/deis/logger/storage/file"
6+
"github.com/deis/logger/storage/redis"
67
"github.com/deis/logger/storage/ringbuffer"
78
)
89

@@ -23,5 +24,12 @@ func NewAdapter(storeageAdapterType string, numLines int) (Adapter, error) {
2324
}
2425
return adapter, nil
2526
}
27+
if storeageAdapterType == "redis" {
28+
adapter, err := redis.NewStorageAdapter(numLines)
29+
if err != nil {
30+
return nil, err
31+
}
32+
return adapter, nil
33+
}
2634
return nil, fmt.Errorf("Unrecognized storage adapter type: '%s'", storeageAdapterType)
2735
}

‎storage/factory_test.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,18 @@ func TestGetMemoryBasedAdapter(t *testing.T) {
3838
}
3939
}
4040

41+
func TestGetRedisBasedAdapter(t *testing.T) {
42+
a, err := NewAdapter("redis", 1)
43+
if err != nil {
44+
t.Error(err)
45+
}
46+
expected := "*redis.adapter"
47+
aType := reflect.TypeOf(a).String()
48+
if aType != expected {
49+
t.Errorf("Expected a %s, but got a %s", expected, aType)
50+
}
51+
}
52+
4153
func TestMain(m *testing.M) {
4254
os.Exit(m.Run())
4355
}

‎storage/redis/adapter.go‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package redis
2+
3+
import (
4+
"fmt"
5+
"log"
6+
7+
r "gopkg.in/redis.v3"
8+
)
9+
10+
type adapter struct {
11+
bufferSize int
12+
redisClient *r.Client
13+
}
14+
15+
// NewStorageAdapter returns a pointer to a new instance of a redis-based storage.Adapter.
16+
func NewStorageAdapter(bufferSize int) (*adapter, error) {
17+
if bufferSize <= 0 {
18+
return nil, fmt.Errorf("Invalid buffer size: %d", bufferSize)
19+
}
20+
cfg, err := parseConfig(appName)
21+
if err != nil {
22+
log.Fatalf("config error: %s: ", err)
23+
}
24+
if err != nil {
25+
return nil, err
26+
}
27+
return &adapter{
28+
bufferSize: bufferSize,
29+
redisClient: r.NewClient(&r.Options{
30+
Addr: fmt.Sprintf("%s:%d", cfg.RedisHost, cfg.RedisPort),
31+
Password: cfg.RedisPassword, // "" == no password
32+
DB: int64(cfg.RedisDB),
33+
}),
34+
}, nil
35+
}
36+
37+
// Write adds a log message to to an app-specific list in redis using ring-buffer-like semantics
38+
func (a *adapter) Write(app string, message string) error {
39+
// Note: Deliberately NOT using MULTI / transactions here since in this implementation of the
40+
// redis client, MULTI is not safe for concurrent use by multiple goroutines. It's been advised
41+
// by the authors of the gopkg.in/redis.v3 package to just use pipelining when possible...
42+
// and here that is technically possible. In the WORST case scenario, not having transactions
43+
// means we may momentarily have more than the desired number of log entries in the list /
44+
// buffer, but an LTRIM will eventually correct that, bringing the list / buffer back down to
45+
// its desired max size.
46+
pipeline := a.redisClient.Pipeline()
47+
if err := pipeline.RPush(app, message).Err(); err != nil {
48+
return err
49+
}
50+
if err := pipeline.LTrim(app, int64(-1*a.bufferSize), -1).Err(); err != nil {
51+
return err
52+
}
53+
if _, err := pipeline.Exec(); err != nil {
54+
return err
55+
}
56+
return nil
57+
}
58+
59+
// Read retrieves a specified number of log lines from an app-specific list in redis
60+
func (a *adapter) Read(app string, lines int) ([]string, error) {
61+
stringSliceCmd := a.redisClient.LRange(app, int64(-1*lines), -1)
62+
result, err := stringSliceCmd.Result()
63+
if err != nil {
64+
return nil, err
65+
}
66+
if len(result) > 0 {
67+
return result, nil
68+
}
69+
return nil, fmt.Errorf("Could not find logs for '%s'", app)
70+
}
71+
72+
// Destroy deletes an app-specific list from redis
73+
func (a *adapter) Destroy(app string) error {
74+
if err := a.redisClient.Del(app).Err(); err != nil {
75+
return err
76+
}
77+
return nil
78+
}
79+
80+
func (a *adapter) Reopen() error {
81+
// No-op
82+
return nil
83+
}

‎storage/redis/adapter_test.go‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
package redis
2+
3+
import (
4+
"fmt"
5+
"testing"
6+
)
7+
8+
const app string = "test-app"
9+
10+
func TestReadFromNonExistingApp(t *testing.T) {
11+
// Initialize a new storage adapter
12+
a, err := NewStorageAdapter(10)
13+
if err != nil {
14+
t.Error(err)
15+
}
16+
// No logs have been written; there should be no redis list for app
17+
messages, err := a.Read(app, 10)
18+
if messages != nil {
19+
t.Error("Expected no messages, but got some")
20+
}
21+
if err == nil || err.Error() != fmt.Sprintf("Could not find logs for '%s'", app) {
22+
t.Error("Did not receive expected error message")
23+
}
24+
}
25+
26+
func TestWithBadBufferSizes(t *testing.T) {
27+
// Initialize with invalid buffer sizes
28+
for _, size := range []int{-1, 0} {
29+
a, err := NewStorageAdapter(size)
30+
if a != nil {
31+
t.Error("Expected no storage adapter, but got one")
32+
}
33+
if err == nil || err.Error() != fmt.Sprintf("Invalid buffer size: %d", size) {
34+
t.Error("Did not receive expected error message")
35+
}
36+
}
37+
}
38+
39+
func TestLogs(t *testing.T) {
40+
// Initialize with small buffers
41+
a, err := NewStorageAdapter(10)
42+
if err != nil {
43+
t.Error(err)
44+
}
45+
// And write a few logs to it, but do NOT fill it up
46+
for i := 0; i < 5; i++ {
47+
if err := a.Write(app, fmt.Sprintf("message %d", i)); err != nil {
48+
t.Error(err)
49+
}
50+
}
51+
// Read more logs than there are
52+
messages, err := a.Read(app, 8)
53+
if err != nil {
54+
t.Error(err)
55+
}
56+
// Should only get as many messages as we actually have
57+
if len(messages) != 5 {
58+
t.Errorf("only expected 5 log messages, got %d", len(messages))
59+
}
60+
// Read fewer logs than there are
61+
messages, err = a.Read(app, 3)
62+
if err != nil {
63+
t.Error(err)
64+
}
65+
// Should get the 3 MOST RECENT logs
66+
if len(messages) != 3 {
67+
t.Errorf("only expected 5 log messages, got %d", len(messages))
68+
}
69+
for i := 0; i < 3; i++ {
70+
expectedMessage := fmt.Sprintf("message %d", i+2)
71+
if messages[i] != expectedMessage {
72+
t.Errorf("expected: \"%s\", got \"%s\"", expectedMessage, messages[i])
73+
}
74+
}
75+
// Overfill the buffer
76+
for i := 5; i < 11; i++ {
77+
if err := a.Write(app, fmt.Sprintf("message %d", i)); err != nil {
78+
t.Error(err)
79+
}
80+
}
81+
// Read more logs than the buffer can hold
82+
messages, err = a.Read(app, 20)
83+
if err != nil {
84+
t.Error(err)
85+
}
86+
// Should only get as many messages as the buffer can hold
87+
if len(messages) != 10 {
88+
t.Errorf("only expected 10 log messages, got %d", len(messages))
89+
}
90+
// And they should only be the 10 MOST RECENT logs
91+
for i := 0; i < 10; i++ {
92+
expectedMessage := fmt.Sprintf("message %d", i+1)
93+
if messages[i] != expectedMessage {
94+
t.Errorf("expected: \"%s\", got \"%s\"", expectedMessage, messages[i])
95+
}
96+
}
97+
}
98+
99+
func TestDestroy(t *testing.T) {
100+
a, err := NewStorageAdapter(10)
101+
if err != nil {
102+
t.Error(err)
103+
}
104+
// Write a log to create the file
105+
if err := a.Write(app, "Hello, log!"); err != nil {
106+
t.Error(err)
107+
}
108+
// A redis list should exist for the app
109+
exists, err := a.redisClient.Exists(app).Result()
110+
if err != nil {
111+
t.Error(err)
112+
}
113+
if !exists {
114+
t.Error("Log redis list was expected to exist, but doesn't.")
115+
}
116+
// Now destroy it
117+
if err := a.Destroy(app); err != nil {
118+
t.Error(err)
119+
}
120+
// Now check that the redis list no longer exists
121+
exists, err = a.redisClient.Exists(app).Result()
122+
if err != nil {
123+
t.Error(err)
124+
}
125+
if exists {
126+
t.Error("Log redis list still exist, but was expected not to.")
127+
}
128+
}

‎storage/redis/config.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package redis
2+
3+
import (
4+
"github.com/kelseyhightower/envconfig"
5+
)
6+
7+
const (
8+
appName = "logger"
9+
)
10+
11+
type config struct {
12+
RedisHost string `envconfig:"DEIS_LOGGER_REDIS_SERVICE_HOST" default:""`
13+
RedisPort int `envconfig:"DEIS_LOGGER_REDIS_SERVICE_PORT" default:"6379"`
14+
RedisPassword string `envconfig:"DEIS_LOGGER_REDIS_PASSWORD" default:""`
15+
RedisDB int `envconfig:"DEIS_LOGGER_REDIS_DB" default:"0"`
16+
}
17+
18+
func parseConfig(appName string) (*config, error) {
19+
ret := new(config)
20+
if err := envconfig.Process(appName, ret); err != nil {
21+
return nil, err
22+
}
23+
return ret, nil
24+
}

0 commit comments

Comments
 (0)