Merge pull request #1068 from deis/router_tcp_forwarding

feat(router): proxy builder through router

Author: carmstrong

builder/bin/boot

@@ -24,10 +24,11 @@ done
 # wait until etcd has discarded potentially stale values
 sleep $(($ETCD_TTL+1))
 
-# seed initial service configuration if necessary
-if ! etcdctl --no-sync -C $ETCD ls $ETCD_PATH >/dev/null 2>&1 ; then
-	etcdctl --no-sync -C $ETCD mkdir $ETCD_PATH/users >/dev/null 2>&1 || true
-fi
+function etcd_safe_mkdir {
+  etcdctl --no-sync -C $ETCD mkdir $1 >/dev/null 2>&1 || true
+}
+
+etcd_safe_mkdir $ETCD_PATH/users
 
 # wait for confd to run once and install initial templates
 until confd -onetime -node $ETCD -config-file /app/confd.toml; do
@@ -72,7 +73,7 @@ echo deis-builder running...
 if [[ ! -z $PUBLISH ]]; then
 
 	# configure service discovery
-	PORT=${PORT:-2222}
+	PORT=${PORT:-2223}
 	PROTO=${PROTO:-tcp}
 
 	set +e

builder/systemd/deis-builder.service

@@ -9,8 +9,8 @@ TimeoutStartSec=20m
 ExecStartPre=/bin/sh -c "docker history deis/builder >/dev/null || docker pull deis/builder:latest"
 ExecStartPre=/bin/sh -c "docker inspect deis-builder >/dev/null && docker rm -f deis-builder || true"
 ExecStartPre=/bin/sh -c "docker inspect deis-builder-data >/dev/null 2>&1 || docker run --name deis-builder-data -v /var/lib/docker deis/base true"
-ExecStart=/usr/bin/docker run --name deis-builder -p 2222:22 -e PUBLISH=22 -e HOST=${COREOS_PRIVATE_IPV4} -e PORT=2222 --volumes-from deis-builder-data --privileged deis/builder
-ExecStartPost=/bin/sh -c "echo 'Waiting for builder on 2222/tcp...' && until cat </dev/null>/dev/tcp/$COREOS_PRIVATE_IPV4/2222; do sleep 1; done"
+ExecStart=/usr/bin/docker run --name deis-builder -p 2223:22 -e PUBLISH=22 -e HOST=${COREOS_PRIVATE_IPV4} -e PORT=2223 --volumes-from deis-builder-data --privileged deis/builder
+ExecStartPost=/bin/sh -c "echo 'Waiting for builder on 2223/tcp...' && until cat </dev/null>/dev/tcp/$COREOS_PRIVATE_IPV4/2223; do sleep 1; done"
 ExecStop=/usr/bin/docker rm -f deis-builder
 
 [Install]

docs/operations/configure-load-balancers.rst

@@ -6,10 +6,21 @@
 Configure load balancers
 ------------------------
 
-For a one-node Deis cluster, there is one router and one controller, so load balancing is unnecessary. You can proceed with the next section: :ref:`configure-dns`.
+For a one-node Deis cluster, there is one router and one controller, so load balancing is unnecessary.
+You can proceed with the next section: :ref:`configure-dns`.
 
-On a multi-node cluster, however, there are probably multiple routers scheduled to the cluster, and these can potentially move hosts. Therefore, it is recommended that you configure a load balancer to operate in front of the Deis cluster to serve application traffic. A simple configuration is one that has all Deis machines listed in its configuration file, but a host is only considered 'healthy' when it is serving traffic on port 80. This enables the load balancer to serve trafic to whichever hosts happen to be running the deis-router component at any one time.
+On a multi-node cluster, however, there are probably multiple routers scheduled to the cluster, and
+these can potentially move hosts. Therefore, it is recommended that you configure a load balancer
+to operate in front of the Deis cluster to serve application traffic. A simple configuration is one
+that has all Deis machines listed in its configuration file, but a host is only considered 'healthy'
+when it is responding to ports 80 and 2222. This enables the load balancer to serve trafic to whichever
+hosts happen to be running the deis-router component at any one time.
 
-The load balancer is also the suggested SSL termination point, as SSL is not currently supported between Deis components.
+These ports need to be open on the load balancers:
 
-Further documentation around load balancers is planned for Deis 1.0.
+* 80 (for application traffic and for API calls to the controller)
+* 2222 (for traffic to the builder)
+
+Optionally, you can also open port 443 and configure SSL termination on the load balancers, but
+requests should still be forwarded to port 80 on the routers. Communication between Deis components
+is currently unencrypted.

router/Dockerfile

@@ -1,19 +1,28 @@
 FROM deis/base
 MAINTAINER Gabriel Monroy <gabriel@opdemand.com>
 
-# install nginx
-RUN apt-get update && \
-    apt-get install -yq python-software-properties
-RUN add-apt-repository ppa:chris-lea/redis-server -y
-RUN add-apt-repository ppa:nginx/stable -y
 RUN apt-get update
-RUN apt-get install -yq nginx
+RUN apt-get install -yq patch libpcre3 libpcre3-dev libssl-dev libgeoip-dev
+
+RUN wget -q http://nginx.org/download/nginx-1.6.0.tar.gz -O /tmp/nginx-1.6.0.tar.gz
+RUN wget -q https://github.com/yaoweibin/nginx_tcp_proxy_module/archive/v0.4.5.tar.gz -O /tmp/tcp-proxy.tar.gz
+
+WORKDIR /tmp
+
+RUN tar -xzf nginx-1.6.0.tar.gz
+RUN tar -xzf tcp-proxy.tar.gz
+
+WORKDIR /tmp/nginx-1.6.0
+RUN patch -p1 < /tmp/nginx_tcp_proxy_module-0.4.5/tcp.patch
+RUN ./configure --prefix=/var/lib/nginx --sbin-path=/usr/sbin/nginx --conf-path=/etc/nginx/nginx.conf --http-log-path=/var/log/nginx/access.log --error-log-path=/var/log/nginx/error.log --lock-path=/var/lock/nginx.lock --pid-path=/run/nginx.pid --http-client-body-temp-path=/var/lib/nginx/body --http-fastcgi-temp-path=/var/lib/nginx/fastcgi --http-proxy-temp-path=/var/lib/nginx/proxy --http-scgi-temp-path=/var/lib/nginx/scgi --http-uwsgi-temp-path=/var/lib/nginx/uwsgi --with-debug --with-pcre-jit --with-ipv6 --with-http_ssl_module --with-http_stub_status_module --with-http_realip_module --with-http_auth_request_module --with-http_addition_module --with-http_dav_module --with-http_geoip_module --with-http_gzip_static_module --with-http_spdy_module --with-http_sub_module --with-mail --with-mail_ssl_module --add-module=/tmp/nginx_tcp_proxy_module-0.4.5
+RUN make
+RUN make install
 
 # install latest etcdctl including no-sync options
 RUN wget -q https://s3-us-west-2.amazonaws.com/deis/etcdctl.no-sync -O /usr/local/bin/etcdctl
 RUN chmod +x /usr/local/bin/etcdctl
 
 ADD . /app
 WORKDIR /app
-EXPOSE 80
+EXPOSE 80 2222
 CMD ["/app/bin/boot"]

router/bin/boot

@@ -37,6 +37,7 @@ function etcd_safe_set {
 etcd_safe_mkdir /deis/controller
 etcd_safe_mkdir /deis/services
 etcd_safe_mkdir /deis/domains
+etcd_safe_mkdir /deis/builder
 etcd_safe_set port ${PORT:-80}
 etcd_safe_set gzip on
 etcd_safe_set gzipHttpVersion 1.0

router/conf.d/nginx.conf.toml

@@ -9,6 +9,7 @@ keys = [
   "/deis/router",
   "/deis/domains",
   "/deis/controller",
+  "/deis/builder",
 ]
 check_cmd  = "/usr/sbin/nginx -t -c {{ .src }}"
 reload_cmd = "/usr/sbin/nginx -s reload"

router/systemd/deis-router.service

@@ -6,7 +6,7 @@ EnvironmentFile=/etc/environment
 TimeoutStartSec=20m
 ExecStartPre=/bin/sh -c "docker history deis/router >/dev/null || docker pull deis/router:latest"
 ExecStartPre=/bin/sh -c "docker inspect deis-router >/dev/null && docker rm -f deis-router || true"
-ExecStart=/usr/bin/docker run --name deis-router -p 80:80 -e PUBLISH=80 -e HOST=${COREOS_PRIVATE_IPV4} deis/router
+ExecStart=/usr/bin/docker run --name deis-router -p 80:80 -p 2222:2222 -e PUBLISH=80 -e HOST=${COREOS_PRIVATE_IPV4} deis/router
 ExecStop=/usr/bin/docker rm -f deis-router
 
 [Install]

router/templates/nginx.conf

@@ -49,9 +49,9 @@ http {
             proxy_set_header            Host $host;
             proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_redirect              off;
-            proxy_connect_timeout       10;
-            proxy_send_timeout          30;
-            proxy_read_timeout          30;
+            proxy_connect_timeout       10s;
+            proxy_send_timeout          30s;
+            proxy_read_timeout          30s;
 
             proxy_pass                  http://deis-controller;
         }
@@ -75,12 +75,33 @@ http {
             proxy_set_header            Host $host;
             proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_redirect              off;
-            proxy_connect_timeout       10;
-            proxy_send_timeout          30;
-            proxy_read_timeout          30;
+            proxy_connect_timeout       10s;
+            proxy_send_timeout          30s;
+            proxy_read_timeout          30s;
 
             proxy_pass                  http://{{ Base $service.Key }};
         }
     }
     {{ end }}{{ end }}
 }
+
+tcp {
+    access_log /dev/stdout;
+    tcp_nodelay on;
+    timeout 30000;
+
+    # same directive names, but these are in miliseconds...
+    proxy_connect_timeout       10000;
+    proxy_send_timeout          30000;
+    proxy_read_timeout          30000;
+
+    upstream builder {
+        server {{ .deis_builder_host }}:{{ .deis_builder_port }};
+    }
+
+    server {
+        listen 2222;
+        proxy_pass builder;
+    }
+}
+