feat(router): proxy builder through router

The router component is to be the point of entry for the entire Deis cluster.
This commit makes the router forward connections on port 2222 to the builder.

This requires manually compiling nginx from source with the tcp_proxy module
and adding an appropriate upstream for builder.

TESTING: rebuild the router and builder (both need to be resubmitted with fleet)
```console
$ make -C router uninstall build run
$ make -C builder uninstall build run
```

And focus testing in two areas:
* That nothing was broken with the change to custom-built nginx (use Deis)
* That the router is properly proxying git/SSH (do lots of `git push`es)

closes #535

Author: carmstrong

builder/bin/boot

@@ -24,10 +24,11 @@ done
 # wait until etcd has discarded potentially stale values
 sleep $(($ETCD_TTL+1))
 
-# seed initial service configuration if necessary
-if ! etcdctl --no-sync -C $ETCD ls $ETCD_PATH >/dev/null 2>&1 ; then
-	etcdctl --no-sync -C $ETCD mkdir $ETCD_PATH/users >/dev/null 2>&1 || true
-fi
+function etcd_safe_mkdir {
+  etcdctl --no-sync -C $ETCD mkdir $1 >/dev/null 2>&1 || true
+}
+
+etcd_safe_mkdir $ETCD_PATH/users
 
 # wait for confd to run once and install initial templates
 until confd -onetime -node $ETCD -config-file /app/confd.toml; do
@@ -72,7 +73,7 @@ echo deis-builder running...
 if [[ ! -z $PUBLISH ]]; then
 
 	# configure service discovery
-	PORT=${PORT:-2222}
+	PORT=${PORT:-2223}
 	PROTO=${PROTO:-tcp}
 
 	set +e

builder/systemd/deis-builder.service

@@ -9,8 +9,8 @@ TimeoutStartSec=20m
 ExecStartPre=/bin/sh -c "docker history deis/builder >/dev/null || docker pull deis/builder:latest"
 ExecStartPre=/bin/sh -c "docker inspect deis-builder >/dev/null && docker rm -f deis-builder || true"
 ExecStartPre=/bin/sh -c "docker inspect deis-builder-data >/dev/null 2>&1 || docker run --name deis-builder-data -v /var/lib/docker deis/base true"
-ExecStart=/usr/bin/docker run --name deis-builder -p 2222:22 -e PUBLISH=22 -e HOST=${COREOS_PRIVATE_IPV4} -e PORT=2222 --volumes-from deis-builder-data --privileged deis/builder
-ExecStartPost=/bin/sh -c "echo 'Waiting for builder on 2222/tcp...' && until cat </dev/null>/dev/tcp/$COREOS_PRIVATE_IPV4/2222; do sleep 1; done"
+ExecStart=/usr/bin/docker run --name deis-builder -p 2223:22 -e PUBLISH=22 -e HOST=${COREOS_PRIVATE_IPV4} -e PORT=2223 --volumes-from deis-builder-data --privileged deis/builder
+ExecStartPost=/bin/sh -c "echo 'Waiting for builder on 2223/tcp...' && until cat </dev/null>/dev/tcp/$COREOS_PRIVATE_IPV4/2223; do sleep 1; done"
 ExecStop=/usr/bin/docker rm -f deis-builder
 
 [Install]

docs/operations/configure-load-balancers.rst

@@ -6,10 +6,21 @@
 Configure load balancers
 ------------------------
 
-For a one-node Deis cluster, there is one router and one controller, so load balancing is unnecessary. You can proceed with the next section: :ref:`configure-dns`.
+For a one-node Deis cluster, there is one router and one controller, so load balancing is unnecessary.
+You can proceed with the next section: :ref:`configure-dns`.
 
-On a multi-node cluster, however, there are probably multiple routers scheduled to the cluster, and these can potentially move hosts. Therefore, it is recommended that you configure a load balancer to operate in front of the Deis cluster to serve application traffic. A simple configuration is one that has all Deis machines listed in its configuration file, but a host is only considered 'healthy' when it is serving traffic on port 80. This enables the load balancer to serve trafic to whichever hosts happen to be running the deis-router component at any one time.
+On a multi-node cluster, however, there are probably multiple routers scheduled to the cluster, and
+these can potentially move hosts. Therefore, it is recommended that you configure a load balancer
+to operate in front of the Deis cluster to serve application traffic. A simple configuration is one
+that has all Deis machines listed in its configuration file, but a host is only considered 'healthy'
+when it is responding to ports 80 and 2222. This enables the load balancer to serve trafic to whichever
+hosts happen to be running the deis-router component at any one time.
 
-The load balancer is also the suggested SSL termination point, as SSL is not currently supported between Deis components.
+These ports need to be open on the load balancers:
 
-Further documentation around load balancers is planned for Deis 1.0.
+* 80 (for application traffic and for API calls to the controller)
+* 2222 (for traffic to the builder)
+
+Optionally, you can also open port 443 and configure SSL termination on the load balancers, but
+requests should still be forwarded to port 80 on the routers. Communication between Deis components
+is currently unencrypted.

router/Dockerfile

@@ -1,19 +1,28 @@
 FROM deis/base
 MAINTAINER Gabriel Monroy <gabriel@opdemand.com>
 
-# install nginx
-RUN apt-get update && \
-    apt-get install -yq python-software-properties
-RUN add-apt-repository ppa:chris-lea/redis-server -y
-RUN add-apt-repository ppa:nginx/stable -y
 RUN apt-get update
-RUN apt-get install -yq nginx
+RUN apt-get install -yq patch libpcre3 libpcre3-dev libssl-dev libgeoip-dev
+
+RUN wget -q http://nginx.org/download/nginx-1.6.0.tar.gz -O /tmp/nginx-1.6.0.tar.gz
+RUN wget -q https://github.com/yaoweibin/nginx_tcp_proxy_module/archive/v0.4.5.tar.gz -O /tmp/tcp-proxy.tar.gz
+
+WORKDIR /tmp
+
+RUN tar -xzf nginx-1.6.0.tar.gz
+RUN tar -xzf tcp-proxy.tar.gz
+
+WORKDIR /tmp/nginx-1.6.0
+RUN patch -p1 < /tmp/nginx_tcp_proxy_module-0.4.5/tcp.patch
+RUN ./configure --prefix=/var/lib/nginx --sbin-path=/usr/sbin/nginx --conf-path=/etc/nginx/nginx.conf --http-log-path=/var/log/nginx/access.log --error-log-path=/var/log/nginx/error.log --lock-path=/var/lock/nginx.lock --pid-path=/run/nginx.pid --http-client-body-temp-path=/var/lib/nginx/body --http-fastcgi-temp-path=/var/lib/nginx/fastcgi --http-proxy-temp-path=/var/lib/nginx/proxy --http-scgi-temp-path=/var/lib/nginx/scgi --http-uwsgi-temp-path=/var/lib/nginx/uwsgi --with-debug --with-pcre-jit --with-ipv6 --with-http_ssl_module --with-http_stub_status_module --with-http_realip_module --with-http_auth_request_module --with-http_addition_module --with-http_dav_module --with-http_geoip_module --with-http_gzip_static_module --with-http_spdy_module --with-http_sub_module --with-mail --with-mail_ssl_module --add-module=/tmp/nginx_tcp_proxy_module-0.4.5
+RUN make
+RUN make install
 
 # install latest etcdctl including no-sync options
 RUN wget -q https://s3-us-west-2.amazonaws.com/deis/etcdctl.no-sync -O /usr/local/bin/etcdctl
 RUN chmod +x /usr/local/bin/etcdctl
 
 ADD . /app
 WORKDIR /app
-EXPOSE 80
+EXPOSE 80 2222
 CMD ["/app/bin/boot"]

router/bin/boot

@@ -37,6 +37,7 @@ function etcd_safe_set {
 etcd_safe_mkdir /deis/controller
 etcd_safe_mkdir /deis/services
 etcd_safe_mkdir /deis/domains
+etcd_safe_mkdir /deis/builder
 etcd_safe_set port ${PORT:-80}
 etcd_safe_set gzip on
 etcd_safe_set gzipHttpVersion 1.0

router/conf.d/nginx.conf.toml

@@ -9,6 +9,7 @@ keys = [
   "/deis/router",
   "/deis/domains",
   "/deis/controller",
+  "/deis/builder",
 ]
 check_cmd  = "/usr/sbin/nginx -t -c {{ .src }}"
 reload_cmd = "/usr/sbin/nginx -s reload"

router/systemd/deis-router.service

@@ -6,7 +6,7 @@ EnvironmentFile=/etc/environment
 TimeoutStartSec=20m
 ExecStartPre=/bin/sh -c "docker history deis/router >/dev/null || docker pull deis/router:latest"
 ExecStartPre=/bin/sh -c "docker inspect deis-router >/dev/null && docker rm -f deis-router || true"
-ExecStart=/usr/bin/docker run --name deis-router -p 80:80 -e PUBLISH=80 -e HOST=${COREOS_PRIVATE_IPV4} deis/router
+ExecStart=/usr/bin/docker run --name deis-router -p 80:80 -p 2222:2222 -e PUBLISH=80 -e HOST=${COREOS_PRIVATE_IPV4} deis/router
 ExecStop=/usr/bin/docker rm -f deis-router
 
 [Install]

router/templates/nginx.conf

@@ -49,9 +49,9 @@ http {
             proxy_set_header            Host $host;
             proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_redirect              off;
-            proxy_connect_timeout       10;
-            proxy_send_timeout          30;
-            proxy_read_timeout          30;
+            proxy_connect_timeout       10s;
+            proxy_send_timeout          30s;
+            proxy_read_timeout          30s;
 
             proxy_pass                  http://deis-controller;
         }
@@ -75,12 +75,33 @@ http {
             proxy_set_header            Host $host;
             proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_redirect              off;
-            proxy_connect_timeout       10;
-            proxy_send_timeout          30;
-            proxy_read_timeout          30;
+            proxy_connect_timeout       10s;
+            proxy_send_timeout          30s;
+            proxy_read_timeout          30s;
 
             proxy_pass                  http://{{ Base $service.Key }};
         }
     }
     {{ end }}{{ end }}
 }
+
+tcp {
+    access_log /dev/stdout;
+    tcp_nodelay on;
+    timeout 30000;
+
+    # same directive names, but these are in miliseconds...
+    proxy_connect_timeout       10000;
+    proxy_send_timeout          30000;
+    proxy_read_timeout          30000;
+
+    upstream builder {
+        server {{ .deis_builder_host }}:{{ .deis_builder_port }};
+    }
+
+    server {
+        listen 2222;
+        proxy_pass builder;
+    }
+}
+