Merge pull request #4367 from krancour/kube-proxy-health

feat(router): improve healthcheck

Author: krancour

contrib/aws/deis.template.json

@@ -300,7 +300,7 @@
         "HealthCheck": {
           "HealthyThreshold": "4",
           "Interval": "15",
-          "Target": "TCP:80",
+          "Target": "HTTP:9090/health-check",
           "Timeout": "5",
           "UnhealthyThreshold": "2"
         },
@@ -367,7 +367,8 @@
           {"IpProtocol": "tcp", "FromPort" : "22",  "ToPort" : "22",  "CidrIp" : { "Ref" : "SSHFrom" }},
           {"IpProtocol": "tcp", "FromPort": "80", "ToPort": "80", "SourceSecurityGroupId": { "Ref": "DeisWebELBSecurityGroup" } },
           {"IpProtocol": "tcp", "FromPort": "443", "ToPort": "443", "SourceSecurityGroupId": { "Ref": "DeisWebELBSecurityGroup" } },
-          {"IpProtocol": "tcp", "FromPort": "2222", "ToPort": "2222", "SourceSecurityGroupId": { "Ref": "DeisWebELBSecurityGroup" } }
+          {"IpProtocol": "tcp", "FromPort": "2222", "ToPort": "2222", "SourceSecurityGroupId": { "Ref": "DeisWebELBSecurityGroup" } },
+          {"IpProtocol": "tcp", "FromPort": "9090", "ToPort": "9090", "SourceSecurityGroupId": { "Ref": "DeisWebELBSecurityGroup" } }
         ],
         "VpcId" : { "Ref" : "VPC" }
       }

deisctl/units/deis-kube-proxy.service

@@ -6,7 +6,7 @@ Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 EnvironmentFile=/etc/environment
 ExecStartPre=/bin/bash -c "/opt/bin/download-k8s-binary kube-proxy"
 ExecStartPre=/bin/bash -c "/opt/bin/wupiao $(/usr/bin/etcdctl get /deis/scheduler/k8s/master):8080"
-ExecStart=/bin/bash -c '/opt/bin/kube-proxy --master=`/usr/bin/etcdctl get /deis/scheduler/k8s/master`:8080 --logtostderr=true'
+ExecStart=/bin/bash -c '/opt/bin/kube-proxy --master=`/usr/bin/etcdctl get /deis/scheduler/k8s/master`:8080 --logtostderr=true --healthz-bind-address=0.0.0.0'
 Restart=always
 RestartSec=10
 SuccessExitStatus=2

deisctl/units/deis-router.service

@@ -7,7 +7,7 @@ TimeoutStartSec=20m
 ExecStartPre=-/usr/bin/etcdctl mkdir /registry/services/ >/dev/null 2>&1
 ExecStartPre=/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/router` && docker history $IMAGE >/dev/null 2>&1 || docker pull $IMAGE"
 ExecStartPre=/bin/sh -c "docker inspect deis-router >/dev/null 2>&1 && docker rm -f deis-router || true"
-ExecStart=/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/router` && docker run --name deis-router --rm -p 80:80 -p 2222:2222 -p 443:443 -e EXTERNAL_PORT=80 -e HOST=$COREOS_PRIVATE_IPV4 $IMAGE"
+ExecStart=/bin/sh -c "IMAGE=`/run/deis/bin/get_image /deis/router` && docker run --name deis-router --rm -p 80:80 -p 2222:2222 -p 443:443 -p 9090:9090 -e EXTERNAL_PORT=80 -e HOST=$COREOS_PRIVATE_IPV4 $IMAGE"
 ExecStop=-/usr/bin/docker stop deis-router
 Restart=on-failure
 RestartSec=5

router/Dockerfile

@@ -25,6 +25,6 @@ COPY rootfs /
 RUN build
 
 CMD ["boot"]
-EXPOSE 80 2222
+EXPOSE 80 2222 9090
 
 ENV DEIS_RELEASE 1.11.0-dev

router/rootfs/etc/confd/templates/nginx.conf

@@ -369,13 +369,19 @@ http {
     }{{ end }}
     ## end service definitions for each application
 
-    # healthcheck
+    {{ $scheduler := or (getv "/deis/controller/schedulerModule") "fleet" }}
+
+    # default server, including "classic" healthcheck
     server {
         listen 80 default_server reuseport{{ if ne $useProxyProtocol "false" }} proxy_protocol{{ end }};
         location /health-check {
-            default_type 'text/plain';
             access_log off;
+            {{ if eq $scheduler "k8s" }}
+            proxy_pass http://{{ getenv "HOST" }}:10249/healthz;
+            {{ else }}
+            default_type 'text/plain';
             return 200;
+            {{ end }}
         }
         location /router-nginx-status {
             vhost_traffic_status_display;
@@ -385,6 +391,24 @@ http {
             return 404;
         }
     }
+
+    # healthcheck on 9090 -- never uses proxy_protocol
+    server {
+        listen 9090 default_server;
+        location /health-check {
+            access_log off;
+            {{ if eq $scheduler "k8s" }}
+            proxy_pass http://{{ getenv "HOST" }}:10249/healthz;
+            {{ else }}
+            default_type 'text/plain';
+            return 200;
+            {{ end }}
+        }
+        location / {
+            return 404;
+        }
+    }
+
     #start k8s apps
     {{ range $k8namespace := lsdir "/registry/services/specs/" }}
     {{ $k8appdir := printf "/registry/services/specs/%s" $k8namespace}}{{ range $kapp := ls $k8appdir }}