Skip to content

Commit e8aa998

Browse files
johanneswuerbachjohanneswuerbach
committed
fix(*): Announce the published port to etcd
Announce the external instead of the internal port to etcd. Currently all components are using the same internal and external port, so this never broke deis. Before publishing etcd, scan whether the internal port is opened Builder listens on 22 (internal) and is published on 2223
1 parent 8c02217 commit e8aa998

15 files changed

Lines changed: 84 additions & 38 deletions

File tree

‎builder/bin/boot‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -84,18 +84,18 @@ echo deis-builder running...
8484
if [[ ! -z $PUBLISH ]]; then
8585

8686
# configure service discovery
87-
PORT=${PORT:-2223}
87+
PORT=${PORT:-22}
8888
PROTO=${PROTO:-tcp}
8989

9090
set +e
9191

9292
# wait for the service to become available on PUBLISH port
93-
sleep 1 && while [[ -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do sleep 1; done
93+
sleep 1 && while [[ -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PORT\" && \$1 ~ \"$PROTO.?\"") ]] ; do sleep 1; done
9494

9595
# while the port is listening, publish to etcd
96-
while [[ ! -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do
96+
while [[ ! -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PORT\" && \$1 ~ \"$PROTO.?\"") ]] ; do
9797
etcdctl --no-sync -C $ETCD set $ETCD_PATH/host $HOST --ttl $ETCD_TTL >/dev/null
98-
etcdctl --no-sync -C $ETCD set $ETCD_PATH/port $PORT --ttl $ETCD_TTL >/dev/null
98+
etcdctl --no-sync -C $ETCD set $ETCD_PATH/port $PUBLISH --ttl $ETCD_TTL >/dev/null
9999
sleep $(($ETCD_TTL/2)) # sleep for half the TTL
100100
done
101101

‎builder/tests/builder_test.go‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ func TestBuilder(t *testing.T) {
3838
defer cli.CmdRm("-f", etcdName)
3939
handler := etcdutils.InitEtcd(setdir, setkeys, etcdPort)
4040
etcdutils.PublishEtcd(t, handler)
41-
ipaddr, port := utils.HostAddress(), utils.RandomPort()
42-
fmt.Printf("--- Run deis/builder:%s at %s:%s\n", tag, ipaddr, port)
41+
host, port := utils.HostAddress(), utils.RandomPort()
42+
fmt.Printf("--- Run deis/builder:%s at %s:%s\n", tag, host, port)
4343
name := "deis-builder-" + tag
4444
defer cli.CmdRm("-f", name)
4545
go func() {
@@ -48,18 +48,21 @@ func TestBuilder(t *testing.T) {
4848
"--name", name,
4949
"--rm",
5050
"-p", port+":22",
51-
"-e", "PUBLISH=22",
51+
"-e", "PORT=22",
5252
"-e", "STORAGE_DRIVER=aufs",
53-
"-e", "HOST="+ipaddr,
53+
"-e", "HOST="+host,
5454
"-e", "ETCD_PORT="+etcdPort,
55-
"-e", "PORT="+port,
55+
"-e", "PUBLISH="+port,
5656
"--privileged", "deis/builder:"+tag)
5757
}()
5858
dockercli.PrintToStdout(t, stdout, stdoutPipe, "deis-builder running")
5959
if err != nil {
6060
t.Fatal(err)
6161
}
62-
// TODO: builder needs a few seconds to wake up here--fixme!
62+
// FIXME: builder needs a few seconds to wake up here!
63+
// FIXME: Wait until etcd keys are published
6364
time.Sleep(5000 * time.Millisecond)
6465
dockercli.DeisServiceTest(t, name, port, "tcp")
66+
etcdutils.VerifyEtcdValue(t, "/deis/builder/host", host, etcdPort)
67+
etcdutils.VerifyEtcdValue(t, "/deis/builder/port", port, etcdPort)
6568
}

‎cache/bin/boot‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,12 @@ if [[ ! -z $PUBLISH ]]; then
5555
set +e
5656

5757
# wait for the service to become available on PUBLISH port
58-
sleep 1 && while [[ -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do sleep 1; done
58+
sleep 1 && while [[ -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PORT\" && \$1 ~ \"$PROTO.?\"") ]] ; do sleep 1; done
5959

6060
# while the port is listening, publish to etcd
61-
while [[ ! -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do
61+
while [[ ! -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PORT\" && \$1 ~ \"$PROTO.?\"") ]] ; do
6262
etcdctl --no-sync -C $ETCD set $ETCD_PATH/host $HOST --ttl $ETCD_TTL >/dev/null
63-
etcdctl --no-sync -C $ETCD set $ETCD_PATH/port $PORT --ttl $ETCD_TTL >/dev/null
63+
etcdctl --no-sync -C $ETCD set $ETCD_PATH/port $PUBLISH --ttl $ETCD_TTL >/dev/null
6464
sleep $(($ETCD_TTL/2)) # sleep for half the TTL
6565
done
6666

‎cache/tests/cache_test.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@ package tests
33
import (
44
"fmt"
55
"testing"
6+
"time"
67

78
"github.com/deis/deis/tests/dockercli"
9+
"github.com/deis/deis/tests/etcdutils"
810
"github.com/deis/deis/tests/utils"
911
)
1012

@@ -16,8 +18,8 @@ func TestCache(t *testing.T) {
1618
cli, stdout, stdoutPipe := dockercli.NewClient()
1719
dockercli.RunTestEtcd(t, etcdName, etcdPort)
1820
defer cli.CmdRm("-f", etcdName)
19-
ipaddr, port := utils.HostAddress(), utils.RandomPort()
20-
fmt.Printf("--- Run deis/cache:%s at %s:%s\n", tag, ipaddr, port)
21+
host, port := utils.HostAddress(), utils.RandomPort()
22+
fmt.Printf("--- Run deis/cache:%s at %s:%s\n", tag, host, port)
2123
name := "deis-cache-" + tag
2224
defer cli.CmdRm("-f", name)
2325
go func() {
@@ -27,13 +29,17 @@ func TestCache(t *testing.T) {
2729
"--rm",
2830
"-p", port+":6379",
2931
"-e", "PUBLISH="+port,
30-
"-e", "HOST="+ipaddr,
32+
"-e", "HOST="+host,
3133
"-e", "ETCD_PORT="+etcdPort,
3234
"deis/cache:"+tag)
3335
}()
3436
dockercli.PrintToStdout(t, stdout, stdoutPipe, "started")
3537
if err != nil {
3638
t.Fatal(err)
3739
}
40+
// FIXME: Wait until etcd keys are published
41+
time.Sleep(5000 * time.Millisecond)
3842
dockercli.DeisServiceTest(t, name, port, "tcp")
43+
etcdutils.VerifyEtcdValue(t, "/deis/cache/host", host, etcdPort)
44+
etcdutils.VerifyEtcdValue(t, "/deis/cache/port", port, etcdPort)
3945
}

‎controller/bin/boot‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,12 +82,12 @@ if [[ ! -z $PUBLISH ]]; then
8282
set +e
8383

8484
# wait for the service to become available on PUBLISH port
85-
sleep 1 && while [[ -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do sleep 1; done
85+
sleep 1 && while [[ -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PORT\" && \$1 ~ \"$PROTO.?\"") ]] ; do sleep 1; done
8686

8787
# while the port is listening, publish to etcd
88-
while [[ ! -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do
88+
while [[ ! -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PORT\" && \$1 ~ \"$PROTO.?\"") ]] ; do
8989
etcdctl --no-sync -C $ETCD set $ETCD_PATH/host $HOST --ttl $ETCD_TTL >/dev/null
90-
etcdctl --no-sync -C $ETCD set $ETCD_PATH/port $PORT --ttl $ETCD_TTL >/dev/null
90+
etcdctl --no-sync -C $ETCD set $ETCD_PATH/port $PUBLISH --ttl $ETCD_TTL >/dev/null
9191
sleep $(($ETCD_TTL/2)) # sleep for half the TTL
9292
done
9393

‎controller/tests/controller_test.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package tests
33
import (
44
"fmt"
55
"testing"
6+
"time"
67

78
"github.com/deis/deis/tests/dockercli"
89
"github.com/deis/deis/tests/etcdutils"
@@ -54,5 +55,9 @@ func TestController(t *testing.T) {
5455
if err != nil {
5556
t.Fatal(err)
5657
}
58+
// FIXME: Wait until etcd keys are published
59+
time.Sleep(5000 * time.Millisecond)
5760
dockercli.DeisServiceTest(t, name, port, "http")
61+
etcdutils.VerifyEtcdValue(t, "/deis/controller/host", host, etcdPort)
62+
etcdutils.VerifyEtcdValue(t, "/deis/controller/port", port, etcdPort)
5863
}

‎database/bin/boot‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -85,12 +85,12 @@ if [[ ! -z $PUBLISH ]]; then
8585
set +e
8686

8787
# wait for the service to become available on PUBLISH port
88-
sleep 1 && while [[ -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do sleep 1; done
88+
sleep 1 && while [[ -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PORT\" && \$1 ~ \"$PROTO.?\"") ]] ; do sleep 1; done
8989

9090
# while the port is listening, publish to etcd
91-
while [[ ! -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do
91+
while [[ ! -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PORT\" && \$1 ~ \"$PROTO.?\"") ]] ; do
9292
etcdctl --no-sync -C $ETCD set $ETCD_PATH/host $HOST --ttl $ETCD_TTL >/dev/null
93-
etcdctl --no-sync -C $ETCD set $ETCD_PATH/port $PORT --ttl $ETCD_TTL >/dev/null
93+
etcdctl --no-sync -C $ETCD set $ETCD_PATH/port $PUBLISH --ttl $ETCD_TTL >/dev/null
9494
sleep $(($ETCD_TTL/2)) # sleep for half the TTL
9595
done
9696

‎database/tests/database_test.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@ package tests
33
import (
44
"fmt"
55
"testing"
6+
"time"
67

78
"github.com/deis/deis/tests/dockercli"
9+
"github.com/deis/deis/tests/etcdutils"
810
"github.com/deis/deis/tests/utils"
911
)
1012

@@ -37,6 +39,9 @@ func TestDatabase(t *testing.T) {
3739
if err != nil {
3840
t.Fatal(err)
3941
}
40-
dockercli.DeisServiceTest(
41-
t, "deis-database-"+tag, port, "tcp")
42+
// FIXME: Wait until etcd keys are published
43+
time.Sleep(5000 * time.Millisecond)
44+
dockercli.DeisServiceTest(t, name, port, "tcp")
45+
etcdutils.VerifyEtcdValue(t, "/deis/database/host", host, etcdPort)
46+
etcdutils.VerifyEtcdValue(t, "/deis/database/port", port, etcdPort)
4247
}

‎logger/bin/boot‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,12 @@ if [[ ! -z $PUBLISH ]]; then
4242
set +e
4343

4444
# wait for the service to become available on PUBLISH port
45-
sleep 1 && while [[ -z $(netstat -lnu | awk "\$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do sleep 1; done
45+
sleep 1 && while [[ -z $(netstat -lnu | awk "\$4 ~ \".$PORT\" && \$1 ~ \"$PROTO.?\"") ]] ; do sleep 1; done
4646

4747
# while the port is listening, publish to etcd
48-
while [[ ! -z $(netstat -lnu | awk "\$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do
48+
while [[ ! -z $(netstat -lnu | awk "\$4 ~ \".$PORT\" && \$1 ~ \"$PROTO.?\"") ]] ; do
4949
etcdctl -C $ETCD set --ttl $ETCD_TTL $ETCD_PATH/host $HOST --no-sync >/dev/null
50-
etcdctl -C $ETCD set --ttl $ETCD_TTL $ETCD_PATH/port $PORT --no-sync >/dev/null
50+
etcdctl -C $ETCD set --ttl $ETCD_TTL $ETCD_PATH/port $PUBLISH --no-sync >/dev/null
5151
sleep $(($ETCD_TTL/2)) # sleep for half the TTL
5252
done
5353

‎logger/tests/logger_test.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@ package tests
33
import (
44
"fmt"
55
"testing"
6+
"time"
67

78
"github.com/deis/deis/tests/dockercli"
9+
"github.com/deis/deis/tests/etcdutils"
810
"github.com/deis/deis/tests/utils"
911
)
1012

@@ -17,8 +19,8 @@ func TestLogger(t *testing.T) {
1719
defer cli.CmdRm("-f", etcdName)
1820
dockercli.RunDeisDataTest(t, "--name", "deis-logger-data",
1921
"-v", "/var/log/deis", "deis/base", "/bin/true")
20-
ipaddr, port := utils.HostAddress(), utils.RandomPort()
21-
fmt.Printf("--- Run deis/logger:%s at %s:%s\n", tag, ipaddr, port)
22+
host, port := utils.HostAddress(), utils.RandomPort()
23+
fmt.Printf("--- Run deis/logger:%s at %s:%s\n", tag, host, port)
2224
name := "deis-logger-" + tag
2325
defer cli.CmdRm("-f", name)
2426
go func() {
@@ -28,7 +30,7 @@ func TestLogger(t *testing.T) {
2830
"--rm",
2931
"-p", port+":514/udp",
3032
"-e", "PUBLISH="+port,
31-
"-e", "HOST="+utils.HostAddress(),
33+
"-e", "HOST="+host,
3234
"-e", "ETCD_PORT="+etcdPort,
3335
"--volumes-from", "deis-logger-data",
3436
"deis/logger:"+tag)
@@ -37,5 +39,9 @@ func TestLogger(t *testing.T) {
3739
if err != nil {
3840
t.Fatal(err)
3941
}
42+
// FIXME: Wait until etcd keys are published
43+
time.Sleep(5000 * time.Millisecond)
4044
dockercli.DeisServiceTest(t, name, port, "udp")
45+
etcdutils.VerifyEtcdValue(t, "/deis/logs/host", host, etcdPort)
46+
etcdutils.VerifyEtcdValue(t, "/deis/logs/port", port, etcdPort)
4147
}

0 commit comments

Comments
 (0)