Merge pull request #1101 from deis/resilient_router

fix(router): increase timeouts and survive controller/builder failures

Author: carmstrong

docs/operations/configure-load-balancers.rst

@@ -11,10 +11,7 @@ You can proceed with the next section: :ref:`configure-dns`.
 
 On a multi-node cluster, however, there are probably multiple routers scheduled to the cluster, and
 these can potentially move hosts. Therefore, it is recommended that you configure a load balancer
-to operate in front of the Deis cluster to serve application traffic. A simple configuration is one
-that has all Deis machines listed in its configuration file, but a host is only considered 'healthy'
-when it is responding to ports 80 and 2222. This enables the load balancer to serve trafic to whichever
-hosts happen to be running the deis-router component at any one time.
+to operate in front of the Deis cluster to serve application traffic.
 
 These ports need to be open on the load balancers:
 
@@ -24,3 +21,8 @@ These ports need to be open on the load balancers:
 Optionally, you can also open port 443 and configure SSL termination on the load balancers, but
 requests should still be forwarded to port 80 on the routers. Communication between Deis components
 is currently unencrypted.
+
+A health check should be configured on the load balancer to send an HTTP request to /health-check at
+port 80 on all nodes in the Deis cluster. The health check endpoint returns an HTTP 200. This enables
+the load balancer to serve trafic to whichever hosts happen to be running the deis-router component
+at any moment.

router/templates/nginx.conf

@@ -34,12 +34,13 @@ http {
     access_log /dev/stdout;
     error_log /dev/stdout;
 
+    ## start deis-controller
+    {{ if .deis_controller_host }}
     upstream deis-controller {
         server {{ .deis_controller_host }}:{{ .deis_controller_port }};
     }
 
     server {
-        listen 80 default_server;
         server_name ~^deis\.(?<domain>.+)$;
         server_name_in_redirect off;
         port_in_redirect off;
@@ -50,14 +51,15 @@ http {
             proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_redirect              off;
             proxy_connect_timeout       10s;
-            proxy_send_timeout          30s;
-            proxy_read_timeout          30s;
+            proxy_send_timeout          1200s;
+            proxy_read_timeout          1200s;
 
             proxy_pass                  http://deis-controller;
         }
-    }
+    }{{ end }}
+    ## end deis-controller
 
-    # service definitions for each application
+    ## start service definitions for each application
     {{ $domains := .deis_domains }}{{ range $service := .deis_services }}{{ if $service.Nodes }}
     upstream {{ Base $service.Key }} {
         {{ range $upstream := $service.Nodes }}server {{ $upstream.Value }};
@@ -76,24 +78,39 @@ http {
             proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_redirect              off;
             proxy_connect_timeout       10s;
-            proxy_send_timeout          30s;
-            proxy_read_timeout          30s;
+            proxy_send_timeout          1200s;
+            proxy_read_timeout          1200s;
+
+            add_header                  X-Deis-Upstream $upstream_addr;
 
             proxy_pass                  http://{{ Base $service.Key }};
         }
     }
     {{ end }}{{ end }}
+    ## end service definitions for each application
+
+    # healthcheck
+    server {
+        listen 80 default_server;
+        location /health-check {
+            default_type 'text/plain';
+            access_log off;
+            return 200;
+        }
+    }
 }
 
+## start builder
+{{ if .deis_builder_host }}
 tcp {
     access_log /dev/stdout;
     tcp_nodelay on;
-    timeout 30000;
+    timeout 1200000;
 
     # same directive names, but these are in miliseconds...
     proxy_connect_timeout       10000;
-    proxy_send_timeout          30000;
-    proxy_read_timeout          30000;
+    proxy_send_timeout          1200000;
+    proxy_read_timeout          1200000;
 
     upstream builder {
         server {{ .deis_builder_host }}:{{ .deis_builder_port }};
@@ -103,5 +120,5 @@ tcp {
         listen 2222;
         proxy_pass builder;
     }
-}
-
+}{{ end }}
+## end builder