Merge pull request #3096 from johanneswuerbach/ec2-websockets

johanneswuerbach · johanneswuerbach · commit fa5d1305b299 · 2015-04-29T18:00:13.000+02:00
WebSockets on EC2
diff --git a/contrib/ec2/deis.template.json b/contrib/ec2/deis.template.json
@@ -287,20 +287,29 @@
         "HealthCheck": {
           "HealthyThreshold": "4",
           "Interval": "15",
-          "Target": "HTTP:80/health-check",
+          "Target": "TCP:80",
           "Timeout": "5",
           "UnhealthyThreshold": "2"
         },
         "Subnets": [
           { "Ref" : "Subnet1" },
           { "Ref" : "Subnet2" }
         ],
+        "Policies" : [{
+           "PolicyName" : "EnableProxyProtocol",
+           "PolicyType" : "ProxyProtocolPolicyType",
+           "Attributes" : [{
+              "Name"  : "ProxyProtocol",
+              "Value" : "true"
+           }],
+           "InstancePorts" : ["80", "443"]
+        }],
         "Listeners": [
           {
             "InstancePort": "80",
-            "InstanceProtocol": "HTTP",
+            "InstanceProtocol": "TCP",
             "LoadBalancerPort": "80",
-            "Protocol": "HTTP"
+            "Protocol": "TCP"
           },
           {
             "InstancePort": "443",
diff --git a/contrib/ec2/provision-ec2-cluster.sh b/contrib/ec2/provision-ec2-cluster.sh
@@ -55,15 +55,15 @@ SLEEPTIME=10
 COUNTER=1
 INSTANCE_IDS=""
 until [ $(wc -w <<< $INSTANCE_IDS) -eq $DEIS_NUM_INSTANCES -a "$STACK_STATUS" = "CREATE_COMPLETE" ]; do
-    if [ $COUNTER -gt $ATTEMPTS ]; then 
+    if [ $COUNTER -gt $ATTEMPTS ]; then
         echo "Provisioning instances failed (timeout, $(wc -w <<< $INSTANCE_IDS) of $DEIS_NUM_INSTANCES provisioned after 10m)"
         echo "Destroying stack $STACK_NAME"
         bailout
         exit 1
     fi
 
     STACK_STATUS=$(aws --output text cloudformation describe-stacks --stack-name $STACK_NAME --query 'Stacks[].StackStatus')
-    if [ $STACK_STATUS != "CREATE_IN_PROGRESS" -a $STACK_STATUS != "CREATE_COMPLETE" ] ; then 
+    if [ $STACK_STATUS != "CREATE_IN_PROGRESS" -a $STACK_STATUS != "CREATE_COMPLETE" ] ; then
       echo "error creating stack: "
       aws --output text cloudformation describe-stack-events \
           --stack-name $STACK_NAME \
@@ -127,3 +127,20 @@ echo "Using ELB $ELB_NAME at $ELB_DNS_NAME"
 
 echo_green "Your Deis cluster has been successfully deployed to AWS CloudFormation and is started."
 echo_green "Please continue to follow the instructions in the documentation."
+
+FIRST_INSTANCE=$(aws ec2 describe-instances \
+    --filters Name=tag:aws:cloudformation:stack-name,Values=$STACK_NAME Name=instance-state-name,Values=running \
+    --query 'Reservations[].Instances[].[PublicIpAddress]' \
+    --output text | head -1)
+echo_green "Setting DEISCTL_TUNNEL=$FIRST_INSTANCE"
+export DEISCTL_TUNNEL=$FIRST_INSTANCE
+echo_green "Enabling proxy protocol"
+
+if ! deisctl config router set proxyProtocol=1; then
+    echo_red "#"
+    echo_red "# Enabling proxy protocol failed, please enable proxy protocol "
+    echo_red "# manually after finishing your deis cluster installation."
+    echo_red "#"
+    echo_red "# deisctl config router set proxyProtocol=1"
+    echo_red "#"
+fi
diff --git a/docs/customizing_deis/router_settings.rst b/docs/customizing_deis/router_settings.rst
@@ -65,6 +65,8 @@ setting                                      description
 /deis/router/sslCert                         cluster-wide SSL certificate
 /deis/router/sslKey                          cluster-wide SSL private key
 /deis/router/workerProcesses                 nginx number of worker processes to start (default: auto i.e. available CPU cores)
+/deis/router/proxyProtocol                   nginx PROXY protocol enabled
+/deis/router/proxyRealIpCidr                 nginx IP with CIDR used by the load balancer in front of deis-router (default: 10.0.0.0/8)
 /deis/services/*                             healthy application containers reported by deis/publisher
 /deis/store/gateway/host                     host of the store gateway component (set by store-gateway)
 /deis/store/gateway/port                     port of the store gateway component (set by store-gateway)
@@ -90,3 +92,22 @@ Be sure that your custom image functions in the same way as the `stock router im
 Deis. Specifically, ensure that it sets and reads appropriate etcd keys.
 
 .. _`stock router image`: https://github.com/deis/deis/tree/master/router
+
+PROXY Protocol
+---------------
+PROXY is a simple protocol supported by nginx, HAProxy, Amazon ELB, and others. It provides a method
+to obtain information about the original requests IP address sent to a load
+balancer in front of Deis :ref:`router`.
+
+The Protocol works by prepending, for example, the following to the request:
+
+.. code-block:: text
+
+	PROXY TCP4 129.164.129.164\r\n
+
+The :ref:`router` will pick up the IP information and forward it to the application in the
+``X-Forwarded-For`` header.
+
+Load Balancers supporting the HTTP protocol may not need this, except in cases where one would run
+WebSockets on a Load Balancer without support for WebSockets (for example AWS ELB) and one also
+wants to know the IP address of the original request.
diff --git a/docs/installing_deis/aws.rst b/docs/installing_deis/aws.rst
@@ -189,6 +189,13 @@ Run the cloudformation provision script to spawn a new CoreOS cluster:
     The default name of the CloudFormation stack will be ``deis``. You can specify a different name
     with ``./provision-ec2-cluster.sh <name>``.
 
+Remote IPs behind your ELB
+--------------------------
+
+The ELB you just created is load-balancing raw TCP connections, which is required for custom domain SSL
+and WebSockets. As remote IPs are by default not visible behind a TCP-Proxy, the ELB and your cluster routers
+were created with `Proxy Protocol`_ enabled.
+
 
 Configure DNS
 -------------
@@ -229,3 +236,4 @@ Please reference the AWS documentation for `more information about CloudFormatio
 .. _`PyYAML`: http://pyyaml.org/
 .. _`update_ec2_cluster.sh`: https://github.com/deis/deis/blob/master/contrib/ec2/update-ec2-cluster.sh
 .. _`More information about CloudFormation stack updates`: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks.html
+.. _`Proxy Protocol`: http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/enable-proxy-protocol.html
diff --git a/router/image/templates/deis.conf b/router/image/templates/deis.conf
@@ -1,9 +1,9 @@
 server_name_in_redirect off;
 port_in_redirect off;
-listen 80;
+listen 80{{ if .deis_router_proxyProtocol }} proxy_protocol{{ end }};
 
 {{ if .deis_router_sslCert }}
-listen 443 ssl spdy;
+listen 443 ssl spdy{{ if .deis_router_proxyProtocol }} proxy_protocol{{ end }};
 ssl_certificate /etc/ssl/deis.cert;
 ssl_certificate_key /etc/ssl/deis.key;
 ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
diff --git a/router/image/templates/nginx.conf b/router/image/templates/nginx.conf
@@ -47,7 +47,12 @@ http {
 
     client_max_body_size {{ or (.deis_router_bodySize) "1m" }};
 
-    log_format upstreaminfo '[$time_local] - $remote_addr - $remote_user - $status - "$request" - $bytes_sent - "$http_referer" - "$http_user_agent" - "$server_name" - $upstream_addr - $http_host - $upstream_response_time - $request_time';
+    {{ $useProxyProtocol := or (.deis_router_proxyProtocol) "false" }}{{ if ne $useProxyProtocol "false" }}
+    set_real_ip_from {{ or (.deis_router_proxyRealIpCidr) "10.0.0.0/8" }};
+    real_ip_header proxy_protocol;
+    {{ end }}
+
+    log_format upstreaminfo '[$time_local] - {{ if .deis_router_proxyProtocol }}$proxy_protocol_addr{{ else }}$remote_addr{{ end }} - $remote_user - $status - "$request" - $bytes_sent - "$http_referer" - "$http_user_agent" - "$server_name" - $upstream_addr - $http_host - $upstream_response_time - $request_time';
 
     # send logs to STDOUT so they can be seen using 'docker logs'
     access_log /opt/nginx/logs/access.log upstreaminfo;
@@ -82,7 +87,11 @@ http {
             {{ if eq $useFirewall "true" }}include                     /opt/nginx/firewall/active-mode.rules;{{ end }}
             proxy_buffering             off;
             proxy_set_header            Host $host;
+            {{ if ne $useProxyProtocol "false" }}
+            proxy_set_header            X-Forwarded-For $proxy_protocol_addr;
+            {{ else }}
             proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
+            {{ end }}
             proxy_redirect              off;
             proxy_connect_timeout       {{ or (.deis_router_controller_timeout_connect) "10s" }};
             proxy_send_timeout          {{ or (.deis_router_controller_timeout_send) "20m" }};
@@ -126,7 +135,11 @@ http {
             {{ if eq $useFirewall "true" }}include                     /opt/nginx/firewall/active-mode.rules;{{ end }}
             proxy_buffering             off;
             proxy_set_header            Host $host;
+            {{ if ne $useProxyProtocol "false" }}
+            proxy_set_header            X-Forwarded-For $proxy_protocol_addr;
+            {{ else }}
             proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
+            {{ end }}
             proxy_redirect              off;
             proxy_connect_timeout       10s;
             proxy_send_timeout          {{ $defaultTimeout }}s;
@@ -168,8 +181,8 @@ http {
         {{ if index $root (printf "deis_certs_%s_cert" (Replace (Base $domain.Key) "-" "_" -1)) }}
         server_name_in_redirect off;
         port_in_redirect off;
-        listen 80;
-        listen 443 ssl spdy;
+        listen 80{{ if ne $useProxyProtocol "false" }} proxy_protocol{{ end }};
+        listen 443 ssl spdy{{ if ne $useProxyProtocol "false" }} proxy_protocol{{ end }};
         ssl_certificate /etc/ssl/deis/certs/{{ Base $domain.Key }}.cert;
         ssl_certificate_key /etc/ssl/deis/keys/{{ Base $domain.Key }}.key;
         ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
@@ -194,7 +207,11 @@ http {
             }
             proxy_set_header            X-Forwarded-Port  $access_port;
             proxy_set_header            X-Forwarded-Proto $access_scheme;
+            {{ if ne $useProxyProtocol "false" }}
+            proxy_set_header            X-Forwarded-For   $proxy_protocol_addr;
+            {{ else }}
             proxy_set_header            X-Forwarded-For   $proxy_add_x_forwarded_for;
+            {{ end }}
             proxy_set_header            X-Forwarded-Ssl   $access_ssl;
             proxy_redirect              off;
             proxy_connect_timeout       30s;
@@ -244,7 +261,11 @@ http {
             }
             proxy_set_header            X-Forwarded-Port  $access_port;
             proxy_set_header            X-Forwarded-Proto $access_scheme;
+            {{ if ne $useProxyProtocol "false" }}
+            proxy_set_header            X-Forwarded-For   $proxy_protocol_addr;
+            {{ else }}
             proxy_set_header            X-Forwarded-For   $proxy_add_x_forwarded_for;
+            {{ end }}
             proxy_set_header            X-Forwarded-Ssl   $access_ssl;
             proxy_redirect              off;
             proxy_connect_timeout       30s;
@@ -278,7 +299,7 @@ http {
 
     # healthcheck
     server {
-        listen 80 default_server;
+        listen 80 default_server{{ if .deis_router_proxyProtocol }} proxy_protocol{{ end }};
         location /health-check {
             default_type 'text/plain';
             access_log off;
