Skip to content
This repository was archived by the owner on Aug 17, 2023. It is now read-only.

Commit 2890575

Browse files
author
Jonathan Chauncey
committed
fix(boot): Create better way to generate fluentd conf
closes #31
1 parent 436308a commit 2890575

9 files changed

Lines changed: 215 additions & 193 deletions

File tree

‎manifests/deis-logger-fluentd-daemon.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,10 @@ spec:
3333
env:
3434
- name: DEBUG
3535
value: "true"
36+
- name: SYSLOG_HOST
37+
value: $(DEIS_LOGGER_SERVICE_HOST)
38+
- name: SYSLOG_PORT
39+
value: $(DEIS_LOGGER_SERVICE_PORT_TRANSPORT)
3640
volumes:
3741
- name: varlog
3842
hostPath:

‎rootfs/opt/fluentd/sbin/boot‎

Lines changed: 3 additions & 193 deletions
Original file line numberDiff line numberDiff line change
@@ -1,207 +1,17 @@
11
#!/bin/bash
22
FLUENTD_CONF="/opt/fluentd/conf/fluentd.conf"
3-
FLUENTD_FLUSH_INTERVAL=${FLUENTD_FLUSH_INTERVAL:-10s}
4-
FLUENTD_FLUSH_THREADS=${FLUENTD_FLUSH_THREADS:-1}
5-
FLUENTD_RETRY_LIMIT=${FLUENTD_RETRY_LIMIT:-10}
6-
FLUENTD_DISABLE_RETRY_LIMIT=${FLUENTD_DISABLE_RETRY_LIMIT:-true}
7-
FLUENTD_RETRY_WAIT=${FLUENTD_RETRY_WAIT:-1s}
8-
FLUENTD_MAX_RETRY_WAIT=${FLUENTD_MAX_RETRY_WAIT:-60s}
9-
FLUENTD_BUFFER_CHUNK_LIMIT=${FLUENTD_BUFFER_CHUNK_LIMIT:-8m}
10-
FLUENTD_BUFFER_QUEUE_LIMIT=${FLUENTD_BUFFER_QUEUE_LIMIT:-8192}
11-
FLUENTD_BUFFER_TYPE=${FLUENTD_BUFFER_TYPE:-memory}
12-
FLUENTD_BUFFER_PATH=${FLUENTD_BUFFER_PATH:-/var/fluentd/buffer}
133

14-
if [ -e "/var/log/containers" ]
15-
then
16-
cat << EOF >> $FLUENTD_CONF
17-
<source>
18-
@type tail
19-
path /var/log/containers/*.log
20-
pos_file /var/log/containers.log.pos
21-
time_format %Y-%m-%dT%H:%M:%S.%NZ
22-
tag kubernetes.*
23-
format json
24-
read_from_head false
25-
</source>
26-
27-
# Example:
28-
# Dec 21 23:17:22 gke-foo-1-1-4b5cbd14-node-4eoj startupscript: Finished running startup script /var/run/google.startup.script
29-
<source>
30-
@type tail
31-
format syslog
32-
path /var/log/startupscript.log
33-
pos_file /var/log/startupscript.log.pos
34-
tag startupscript
35-
</source>
36-
37-
# Examples:
38-
# time="2016-02-04T06:51:03.053580605Z" level=info msg="GET /containers/json"
39-
# time="2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No such image: -f" statusCode=404
40-
<source>
41-
@type tail
42-
format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
43-
time_format %Y-%m-%dT%H:%M:%S.%NZ
44-
path /var/log/docker.log
45-
pos_file /var/log/docker.log.pos
46-
tag docker
47-
</source>
48-
49-
# Example:
50-
# 2016/02/04 06:52:38 filePurge: successfully removed file /var/etcd/data/member/wal/00000000000006d0-00000000010a23d1.wal
51-
<source>
52-
@type tail
53-
# Not parsing this, because it doesn't have anything particularly useful to
54-
# parse out of it (like severities).
55-
format none
56-
path /var/log/etcd.log
57-
pos_file /var/log/etcd.log.pos
58-
tag etcd
59-
</source>
60-
61-
# Multi-line parsing is required for all the kube logs because very large log
62-
# statements, such as those that include entire object bodies, get split into
63-
# multiple lines by glog.
64-
65-
# Example:
66-
# I0204 07:32:30.020537 3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
67-
<source>
68-
@type tail
69-
format multiline
70-
format_firstline /^\w\d{4}/
71-
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
72-
time_format %m%d %H:%M:%S.%N
73-
path /var/log/kubelet.log
74-
pos_file /var/log/kubelet.log.pos
75-
tag kubelet
76-
</source>
77-
78-
# Example:
79-
# I0204 07:00:19.604280 5 handlers.go:131] GET /api/v1/nodes: (1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) kubernetes/6a81b50] 127.0.0.1:38266]
80-
<source>
81-
@type tail
82-
format multiline
83-
format_firstline /^\w\d{4}/
84-
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
85-
time_format %m%d %H:%M:%S.%N
86-
path /var/log/kube-apiserver.log
87-
pos_file /var/log/kube-apiserver.log.pos
88-
tag kube-apiserver
89-
</source>
90-
91-
# Example:
92-
# I0204 06:55:31.872680 5 servicecontroller.go:277] LB already exists and doesn't need update for service kube-system/kube-ui
93-
<source>
94-
@type tail
95-
format multiline
96-
format_firstline /^\w\d{4}/
97-
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
98-
time_format %m%d %H:%M:%S.%N
99-
path /var/log/kube-controller-manager.log
100-
pos_file /var/log/kube-controller-manager.log.pos
101-
tag kube-controller-manager
102-
</source>
103-
104-
# Example:
105-
# W0204 06:49:18.239674 7 reflector.go:245] pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [2578313/2577886]) [2579312]
106-
<source>
107-
@type tail
108-
format multiline
109-
format_firstline /^\w\d{4}/
110-
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
111-
time_format %m%d %H:%M:%S.%N
112-
path /var/log/kube-scheduler.log
113-
pos_file /var/log/kube-scheduler.log.pos
114-
tag kube-scheduler
115-
</source>
116-
EOF
117-
else
118-
echo "/var/log/containers does not exist in the container."
119-
fi
120-
121-
122-
if [ -n "$KUBERNETES_SERVICE_HOST" ]
123-
then
124-
cat << EOF >> $FLUENTD_CONF
125-
<filter kubernetes.**>
126-
@type kubernetes_metadata
127-
kubernetes_url https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}
128-
bearer_token_file /var/run/secrets/kubernetes.io/serviceaccount/token
129-
verify_ssl false
130-
</filter>
131-
EOF
132-
fi
4+
source /opt/fluentd/sbin/sources
5+
source /opt/fluentd/sbin/filters/filters
1336

1347
cat << EOF >> $FLUENTD_CONF
1358
<match **>
1369
@type copy
13710
EOF
13811

139-
if [ -n "$ELASTICSEARCH_HOST" ]
140-
then
141-
echo "Starting fluentd with elastic search configuration!"
142-
ELASTICSEARCH_PORT=${ELASTICSEARCH_PORT:-9200}
143-
ELASTICSEARCH_SCHEME=${ELASTICSEARCH_SCHEME:-http}
144-
145-
cat << EOF >> $FLUENTD_CONF
146-
<store>
147-
@type elasticsearch
148-
log_level warn
149-
include_tag_key true
150-
time_key time
151-
host ${ELASTICSEARCH_HOST}
152-
port ${ELASTICSEARCH_PORT}
153-
scheme ${ELASTICSEARCH_SCHEME}
154-
$([ -n "${ELASTICSEARCH_USER}" ] && echo user ${ELASTICSEARCH_USER})
155-
$([ -n "${ELASTICSEARCH_PASSWORD}" ] && echo password ${ELASTICSEARCH_PASSWORD})
156-
buffer_type ${FLUENTD_BUFFER_TYPE}
157-
$([ "${FLUENTD_BUFFER_TYPE}" == "file" ] && echo buffer_path ${FLUENTD_BUFFER_PATH})
158-
buffer_chunk_limit ${FLUENTD_BUFFER_CHUNK_LIMIT}
159-
buffer_queue_limit ${FLUENTD_BUFFER_QUEUE_LIMIT}
160-
flush_interval ${FLUENTD_FLUSH_INTERVAL}
161-
retry_limit ${FLUENTD_RETRY_LIMIT}
162-
$([ "${FLUENTD_DISABLE_RETRY_LIMIT}" == "true" ] && echo disable_retry_limit)
163-
retry_wait ${FLUENTD_RETRY_WAIT}
164-
max_retry_wait ${FLUENTD_MAX_RETRY_WAIT}
165-
num_threads ${FLUENTD_FLUSH_THREADS}
166-
</store>
167-
EOF
168-
fi
169-
170-
NUM=$(env | grep -oE '^(SYSLOG_HOST_\d*)' | wc -l)
171-
for ((i=1; i<=$NUM; i++)) do
172-
host=$(eval "echo \$SYSLOG_HOST_$i")
173-
port=$(eval "echo \$SYSLOG_PORT_$i")
174-
if [ -n "$host" ]
175-
then
176-
echo "Starting fluentd with syslog configuration! -- $host:$port"
177-
cat << EOF >> $FLUENTD_CONF
178-
<store>
179-
@type remote_syslog
180-
host $host
181-
port $port
182-
log_level warn
183-
</store>
184-
EOF
185-
fi
186-
done
187-
188-
if [ -n "$SYSLOG_HOST" ]
189-
then
190-
echo "Starting fluentd with syslog configuration! -- $SYSLOG_HOST:$SYSLOG_PORT"
191-
cat << EOF >> $FLUENTD_CONF
192-
<store>
193-
@type remote_syslog
194-
host $SYSLOG_HOST
195-
port $SYSLOG_PORT
196-
log_level warn
197-
</store>
198-
EOF
199-
fi
12+
source /opt/fluentd/sbin/stores/stores
20013

20114
cat << EOF >> $FLUENTD_CONF
202-
<store>
203-
@type deis
204-
</store>
20515
</match>
20616
EOF
20717

‎rootfs/opt/fluentd/sbin/filters/filters‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#!/bin/bash
2+
source /opt/fluentd/sbin/filters/kubernetes

‎rootfs/opt/fluentd/sbin/filters/kubernetes‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
3+
if [ -n "$KUBERNETES_SERVICE_HOST" ]
4+
then
5+
cat << EOF >> $FLUENTD_CONF
6+
<filter kubernetes.**>
7+
@type kubernetes_metadata
8+
kubernetes_url https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}
9+
bearer_token_file /var/run/secrets/kubernetes.io/serviceaccount/token
10+
verify_ssl false
11+
</filter>
12+
EOF
13+
fi

‎rootfs/opt/fluentd/sbin/sources‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
#!/bin/bash
2+
if [ -e "/var/log/containers" ]
3+
then
4+
cat << EOF >> $FLUENTD_CONF
5+
<source>
6+
@type tail
7+
path /var/log/containers/*.log
8+
pos_file /var/log/containers.log.pos
9+
time_format %Y-%m-%dT%H:%M:%S.%NZ
10+
tag kubernetes.*
11+
format json
12+
read_from_head false
13+
</source>
14+
15+
# Example:
16+
# Dec 21 23:17:22 gke-foo-1-1-4b5cbd14-node-4eoj startupscript: Finished running startup script /var/run/google.startup.script
17+
<source>
18+
@type tail
19+
format syslog
20+
path /var/log/startupscript.log
21+
pos_file /var/log/startupscript.log.pos
22+
tag startupscript
23+
</source>
24+
25+
# Examples:
26+
# time="2016-02-04T06:51:03.053580605Z" level=info msg="GET /containers/json"
27+
# time="2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No such image: -f" statusCode=404
28+
<source>
29+
@type tail
30+
format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
31+
time_format %Y-%m-%dT%H:%M:%S.%NZ
32+
path /var/log/docker.log
33+
pos_file /var/log/docker.log.pos
34+
tag docker
35+
</source>
36+
37+
# Example:
38+
# 2016/02/04 06:52:38 filePurge: successfully removed file /var/etcd/data/member/wal/00000000000006d0-00000000010a23d1.wal
39+
<source>
40+
@type tail
41+
# Not parsing this, because it doesn't have anything particularly useful to
42+
# parse out of it (like severities).
43+
format none
44+
path /var/log/etcd.log
45+
pos_file /var/log/etcd.log.pos
46+
tag etcd
47+
</source>
48+
49+
# Multi-line parsing is required for all the kube logs because very large log
50+
# statements, such as those that include entire object bodies, get split into
51+
# multiple lines by glog.
52+
53+
# Example:
54+
# I0204 07:32:30.020537 3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
55+
<source>
56+
@type tail
57+
format multiline
58+
format_firstline /^\w\d{4}/
59+
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
60+
time_format %m%d %H:%M:%S.%N
61+
path /var/log/kubelet.log
62+
pos_file /var/log/kubelet.log.pos
63+
tag kubelet
64+
</source>
65+
66+
# Example:
67+
# I0204 07:00:19.604280 5 handlers.go:131] GET /api/v1/nodes: (1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) kubernetes/6a81b50] 127.0.0.1:38266]
68+
<source>
69+
@type tail
70+
format multiline
71+
format_firstline /^\w\d{4}/
72+
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
73+
time_format %m%d %H:%M:%S.%N
74+
path /var/log/kube-apiserver.log
75+
pos_file /var/log/kube-apiserver.log.pos
76+
tag kube-apiserver
77+
</source>
78+
79+
# Example:
80+
# I0204 06:55:31.872680 5 servicecontroller.go:277] LB already exists and doesn't need update for service kube-system/kube-ui
81+
<source>
82+
@type tail
83+
format multiline
84+
format_firstline /^\w\d{4}/
85+
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
86+
time_format %m%d %H:%M:%S.%N
87+
path /var/log/kube-controller-manager.log
88+
pos_file /var/log/kube-controller-manager.log.pos
89+
tag kube-controller-manager
90+
</source>
91+
92+
# Example:
93+
# W0204 06:49:18.239674 7 reflector.go:245] pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [2578313/2577886]) [2579312]
94+
<source>
95+
@type tail
96+
format multiline
97+
format_firstline /^\w\d{4}/
98+
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
99+
time_format %m%d %H:%M:%S.%N
100+
path /var/log/kube-scheduler.log
101+
pos_file /var/log/kube-scheduler.log.pos
102+
tag kube-scheduler
103+
</source>
104+
EOF
105+
else
106+
echo "/var/log/containers does not exist in the container."
107+
fi

‎rootfs/opt/fluentd/sbin/stores/deis‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
cat << EOF >> $FLUENTD_CONF
3+
<store>
4+
@type deis
5+
</store>
6+
EOF

‎rootfs/opt/fluentd/sbin/stores/elastic_search‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/bin/bash
2+
FLUENTD_FLUSH_INTERVAL=${FLUENTD_FLUSH_INTERVAL:-10s}
3+
FLUENTD_FLUSH_THREADS=${FLUENTD_FLUSH_THREADS:-1}
4+
FLUENTD_RETRY_LIMIT=${FLUENTD_RETRY_LIMIT:-10}
5+
FLUENTD_DISABLE_RETRY_LIMIT=${FLUENTD_DISABLE_RETRY_LIMIT:-true}
6+
FLUENTD_RETRY_WAIT=${FLUENTD_RETRY_WAIT:-1s}
7+
FLUENTD_MAX_RETRY_WAIT=${FLUENTD_MAX_RETRY_WAIT:-60s}
8+
FLUENTD_BUFFER_CHUNK_LIMIT=${FLUENTD_BUFFER_CHUNK_LIMIT:-8m}
9+
FLUENTD_BUFFER_QUEUE_LIMIT=${FLUENTD_BUFFER_QUEUE_LIMIT:-8192}
10+
FLUENTD_BUFFER_TYPE=${FLUENTD_BUFFER_TYPE:-memory}
11+
FLUENTD_BUFFER_PATH=${FLUENTD_BUFFER_PATH:-/var/fluentd/buffer}
12+
13+
if [ -n "$ELASTICSEARCH_HOST" ]
14+
then
15+
echo "Starting fluentd with elastic search configuration!"
16+
ELASTICSEARCH_PORT=${ELASTICSEARCH_PORT:-9200}
17+
ELASTICSEARCH_SCHEME=${ELASTICSEARCH_SCHEME:-http}
18+
19+
cat << EOF >> $FLUENTD_CONF
20+
<store>
21+
@type elasticsearch
22+
log_level warn
23+
include_tag_key true
24+
time_key time
25+
host ${ELASTICSEARCH_HOST}
26+
port ${ELASTICSEARCH_PORT}
27+
scheme ${ELASTICSEARCH_SCHEME}
28+
$([ -n "${ELASTICSEARCH_USER}" ] && echo user ${ELASTICSEARCH_USER})
29+
$([ -n "${ELASTICSEARCH_PASSWORD}" ] && echo password ${ELASTICSEARCH_PASSWORD})
30+
buffer_type ${FLUENTD_BUFFER_TYPE}
31+
$([ "${FLUENTD_BUFFER_TYPE}" == "file" ] && echo buffer_path ${FLUENTD_BUFFER_PATH})
32+
buffer_chunk_limit ${FLUENTD_BUFFER_CHUNK_LIMIT}
33+
buffer_queue_limit ${FLUENTD_BUFFER_QUEUE_LIMIT}
34+
flush_interval ${FLUENTD_FLUSH_INTERVAL}
35+
retry_limit ${FLUENTD_RETRY_LIMIT}
36+
$([ "${FLUENTD_DISABLE_RETRY_LIMIT}" == "true" ] && echo disable_retry_limit)
37+
retry_wait ${FLUENTD_RETRY_WAIT}
38+
max_retry_wait ${FLUENTD_MAX_RETRY_WAIT}
39+
num_threads ${FLUENTD_FLUSH_THREADS}
40+
</store>
41+
EOF
42+
fi

0 commit comments

Comments
 (0)