ref(router): split domains to its own custom directive

In order to support multiple certs on multiple custom domains bound to
the same application, we need to split out each custom domain to its own
`server {}` directive. In master, the domain etcd keys are set like so:

    SET /deis/domains/go "www.bacongobbler.com foo.bacongobbler.com"

In confd v0.5.0, there is no way to split strings by whitespace so
there's no way to iterate through each domain bound to an application.
In this commit, there's a controller data migration script which will
migrate existing etcd keys for custom domain endpoints over to the new
syntax:

    SET /deis/domains/www.bacongobbler.com "go"
    SET /deis/domains/foo.bacongobbler.com "go"

Confd v0.8.0 has support for splitting strings, but with this approach
we can start providing support for migrations such as the one proposed
in #3399.

Author: Matthew Fisher

controller/api/models.py

@@ -977,29 +977,26 @@ def _etcd_publish_cert(**kwargs):
 
 def _etcd_purge_cert(**kwargs):
     cert = kwargs['instance']
-    _etcd_client.delete('/deis/certs/{}'.format(cert),
-                        prevExist=True, dir=True, recursive=True)
+    try:
+        _etcd_client.delete('/deis/certs/{}'.format(cert),
+                            prevExist=True, dir=True, recursive=True)
+    except KeyError:
+        pass
 
 
 def _etcd_publish_domains(**kwargs):
-    app = kwargs['instance'].app
-    app_domains = app.domain_set.all()
-    if app_domains:
-        _etcd_client.write('/deis/domains/{}'.format(app),
-                           ' '.join(str(d.domain) for d in app_domains))
+    domain = kwargs['instance']
+    if kwargs['created']:
+        _etcd_client.write('/deis/domains/{}'.format(domain), domain.app)
 
 
 def _etcd_purge_domains(**kwargs):
-    app = kwargs['instance'].app
-    app_domains = app.domain_set.all()
-    if app_domains:
-        _etcd_client.write('/deis/domains/{}'.format(app),
-                           ' '.join(str(d.domain) for d in app_domains))
-    else:
-        try:
-            _etcd_client.delete('/deis/domains/{}'.format(app))
-        except KeyError:
-            pass
+    domain = kwargs['instance']
+    try:
+        _etcd_client.delete('/deis/certs/{}'.format(domain),
+                            prevExist=True, dir=True, recursive=True)
+    except KeyError:
+        pass
 
 
 # Log significant app-related events

controller/bin/boot

@@ -56,6 +56,14 @@ etcd_safe_mkdir /deis/services
 etcd_safe_mkdir /deis/domains
 etcd_safe_mkdir /deis/platform
 
+# run etcd data migrations
+echo "controller: running etcd data migrations..."
+for script in $(ls /app/migrations/data/*.sh);
+do
+    . $script;
+done
+echo "controller: done running etcd data migrations."
+
 # wait for confd to run once and install initial templates
 until confd -onetime -node $ETCD --confdir /app --interval 5 --quiet 2>/dev/null; do
 	echo "controller: waiting for confd to write initial templates..."

controller/migrations/data/0001.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+ETCD_PORT=${ETCD_PORT:-4001}
+ETCD="$HOST:$ETCD_PORT"
+ETCDCTL="etcdctl -C $ETCD"
+
+if [[ "$($ETCDCTL get /deis/migrations/data/0001 2> /dev/null)" != "done" ]];
+then
+    for i in $($ETCDCTL ls /deis/domains 2> /dev/null);
+    do
+        for j in $($ETCDCTL get $i);
+        do
+            $ETCDCTL set /deis/domains/$j "$(basename $i)" 1> /dev/null;
+            echo "migrated $j"
+        done;
+        $ETCDCTL rm $i;
+    done
+    $ETCDCTL set /deis/migrations/data/0001 "done"
+fi

router/image/templates/deis.conf

@@ -1,8 +1,8 @@
 server_name_in_redirect off;
 port_in_redirect off;
+listen 80;
 
 {{ if .deis_router_sslCert }}
-listen 80;
 listen 443 ssl spdy;
 ssl_certificate /etc/ssl/deis.cert;
 ssl_certificate_key /etc/ssl/deis.key;

router/image/templates/nginx.conf

@@ -139,10 +139,10 @@ http {
     ## end deis-store-gateway
 
     ## start service definitions for each application
-    {{ $useSSL := or .deis_router_sslCert "false" }}
     {{ $affinityArg := .deis_router_affinityArg }}
     {{ $certs := .deis_certs }}
-    {{ $domains := .deis_domains }}{{ range $service := .deis_services }}{{ if $service.Nodes }}
+    {{ $domains := .deis_domains }}
+    {{ range $service := .deis_services }}{{ if $service.Nodes }}
     upstream {{ Base $service.Key }} {
         {{ if $affinityArg }}hash $arg_{{ $affinityArg }} consistent;
         {{ end }}
@@ -151,21 +151,72 @@ http {
     }
     {{ end }}
 
+    ## server entries for custom domains
+    {{ range $domain := $domains }}{{ if eq (Base $service.Key) $domain.Value }}
     server {
-        server_name ~^{{ Base $service.Key }}\.(?<domain>.+)${{ range $app_domains := $domains }}{{ if eq (Base $service.Key) (Base $app_domains.Key) }} {{ $app_domains.Value }}{{ end }}{{ end }};
-        {{ range $domain := $domains }}{{ range $cert := $certs }}
-        {{ if eq $domain.Value (Base $cert.Key) }}
-        {{ $useSSL := "true" }}
+        server_name {{ Base $domain.Key }};
         server_name_in_redirect off;
         port_in_redirect off;
         listen 80;
+        {{/* if a SSL certificate is installed for this domain, use SSL */}}
+        {{/* Note (bacongobbler): domains are separate from the default platform domain, */}}
+        {{/* so we can't rely on deis.conf as each domain is an island */}}
+        {{ range $cert := $certs }}{{ if eq (Base $domain.Key) (Base $cert.Key) }}
         listen 443 ssl spdy;
-        ssl_certificate /etc/ssl/deis/certs/{{ Base $cert.Key }}.cert;
-        ssl_certificate_key /etc/ssl/deis/keys/{{ Base $cert.Key }}.key;
+        ssl_certificate /etc/ssl/deis/certs/{{ Base $domain.Key }}.cert;
+        ssl_certificate_key /etc/ssl/deis/keys/{{ Base $domain.Key }}.key;
+        ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+        {{ end }}{{ end }}
+
+        {{ if $service.Nodes }}
+        location / {
+            {{ if eq $useFirewall "true" }}include                     /opt/nginx/firewall/active-mode.rules;{{ end }}
+            proxy_buffering             off;
+            proxy_set_header            Host $host;
+            set $access_ssl 'off';
+            set $access_port '80';
+            if ($access_scheme ~ https) {
+                set $access_ssl 'on';
+                set $access_port '443';
+            }
+            proxy_set_header            X-Forwarded-Port  $access_port;
+            proxy_set_header            X-Forwarded-Proto $access_scheme;
+            proxy_set_header            X-Forwarded-For   $proxy_add_x_forwarded_for;
+            proxy_set_header            X-Forwarded-Ssl   $access_ssl;
+            proxy_redirect              off;
+            proxy_connect_timeout       30s;
+            proxy_send_timeout          1200s;
+            proxy_read_timeout          1200s;
+            proxy_http_version          1.1;
+            proxy_set_header            Upgrade           $http_upgrade;
+            proxy_set_header            Connection        $connection_upgrade;
+
+            proxy_next_upstream         error timeout http_502 http_503 http_504;
+
+            {{ if eq $enforceHTTPS "true" }}
+            if ($access_scheme != "https") {
+              return 301 https://$server_name$request_uri;
+            }
+            {{ end }}
+
+            proxy_pass                  http://{{ Base $service.Key }};
+        }
         {{ else }}
-        include deis.conf;
+        location / {
+            return 503;
+        }
         {{ end }}
-        {{ end }}{{ end }}
+
+        {{ if eq $useFirewall "true" }}location /RequestDenied {
+            return {{ $firewallErrorCode }};
+        }{{ end }}
+    }
+    {{ end }}{{ end }}
+    ## end entries for custom domains
+
+    server {
+        server_name ~^{{ Base $service.Key }}\.(?<domain>.+)$;
+        include deis.conf;
 
         {{ if $service.Nodes }}
         location / {