docs(k8s): add k8s integration documentation in customizing schedulers

Author: Keerthan Mala

contrib/coreos/user-data.example

@@ -159,6 +159,9 @@ write_files:
   - path: /etc/deis-release
     content: |
       DEIS_RELEASE=v1.8.0
+  - path: /etc/k8environment
+    content: |
+      K8S_VERSION=v1.0.1
   - path: /etc/motd
     content: " \e[31m* *    \e[34m*   \e[32m*****    \e[39mddddd   eeeeeee iiiiiii   ssss\n\e[31m*   *  \e[34m* *  \e[32m*   *     \e[39md   d   e    e    i     s    s\n \e[31m* *  \e[34m***** \e[32m*****     \e[39md    d  e         i    s\n\e[32m*****  \e[31m* *    \e[34m*       \e[39md     d e         i     s\n\e[32m*   * \e[31m*   *  \e[34m* *      \e[39md     d eee       i      sss\n\e[32m*****  \e[31m* *  \e[34m*****     \e[39md     d e         i         s\n  \e[34m*   \e[32m*****  \e[31m* *      \e[39md    d  e         i          s\n \e[34m* *  \e[32m*   * \e[31m*   *     \e[39md   d   e    e    i    s    s\n\e[34m***** \e[32m*****  \e[31m* *     \e[39mddddd   eeeeeee iiiiiii  ssss\n\n\e[39mWelcome to Deis\t\t\tPowered by Core\e[38;5;45mO\e[38;5;206mS\e[39m\n"
   - path: /etc/profile.d/nse-function.sh
@@ -350,4 +353,89 @@ write_files:
         /bin/openssl genrsa -out /opt/bin/kube-serviceaccount.key 2048 2>/dev/null
         etcdctl set /kube-serviceaccount < /opt/bin/kube-serviceaccount.key
       fi
+  - path: /opt/bin/wupiao
+    permissions: '0755'
+    content: |
+      #!/bin/bash
+      # [w]ait [u]ntil [p]ort [i]s [a]ctually [o]pen
+      [ -n "$1" ] && \
+        until curl -o /dev/null -sIf http://${1}; do \
+          sleep 1 && echo .;
+        done;
+      exit $?
+  - path: /opt/bin/download-k8s-binary
+    permissions: '0755'
+    content: |
+      #!/bin/bash
+
+      . /etc/k8environment
+
+      mkdir -p /opt/bin
+
+      FILE=$1
+      if [ ! -f /opt/bin/$FILE ]; then
+        curl -sSL -o /opt/bin/$FILE https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/$FILE
+        chmod +x /opt/bin/$FILE
+      else
+        # we check the version of the binary
+        INSTALLED_VERSION=$(/opt/bin/$FILE --version)
+        MATCH=$(echo "${INSTALLED_VERSION}" | grep -c "${K8S_VERSION}")
+        if [ $MATCH -eq 0 ]; then
+          # the version is different
+          curl -sSL -o /opt/bin/$FILE https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/$FILE
+          chmod +x /opt/bin/$FILE
+        fi
+      fi
+  - path: /opt/bin/kube-serviceaccount
+    permissions: '0755'
+    content: |
+      #!/bin/bash
+
+      if etcdctl get /kube-serviceaccount >/dev/null 2>&1; then
+        etcdctl get /kube-serviceaccount > /opt/bin/kube-serviceaccount.key
+      else
+        /bin/openssl genrsa -out /opt/bin/kube-serviceaccount.key 2048 2>/dev/null
+        etcdctl set /kube-serviceaccount < /opt/bin/kube-serviceaccount.key
+      fi
+  - path: /opt/bin/scheduler-policy.json
+    content: |
+      {
+          "kind": "Policy",
+          "apiVersion": "v1",
+          "predicates": [
+              {
+                  "name": "PodFitsPorts"
+              },
+              {
+                  "name": "PodFitsResources"
+              },
+              {
+                  "name": "NoDiskConflict"
+              },
+              {
+                  "name": "MatchNodeSelector"
+              },
+              {
+                  "name": "HostName"
+              }
+          ],
+          "priorities": [
+              {
+                  "name": "LeastRequestedPriority",
+                  "weight": 1
+              },
+              {
+                  "name": "BalancedResourceAllocation",
+                  "weight": 1
+              },
+              {
+                  "name": "ServiceSpreadingPriority",
+                  "weight": 2
+              },
+              {
+                  "name": "EqualPriority",
+                  "weight": 1
+              }
+          ]
+      }
 manage_etc_hosts: localhost

controller/scheduler/k8s.py

@@ -39,13 +39,15 @@
       "replicas":$num,
       "selector":{
          "name":"$id",
-         "version":"$appversion"
+         "version":"$appversion",
+         "type":"$type"
       },
       "template":{
          "metadata":{
             "labels":{
                "name":"$id",
-               "version":"$appversion"
+               "version":"$appversion",
+               "type":"$type"
             }
          },
          "spec":{
@@ -78,7 +80,8 @@
         }
       ],
       "selector":{
-         "name":"$label"
+         "name":"$label",
+         "type":"$type"
       }
    }
 }'''
@@ -101,7 +104,7 @@ def __init__(self, target, auth, options, pkey):
         self.apiversion = "v1"
         self.conn = httplib.HTTPConnection(self.target+":"+self.port)
 
-    def _get_old_rc(self, name):
+    def _get_old_rc(self, name, app_type):
         con_app = httplib.HTTPConnection(self.target+":"+self.port)
         con_app.request('GET', '/api/'+self.apiversion +
                         '/namespaces/default/replicationcontrollers')
@@ -118,7 +121,8 @@ def _get_old_rc(self, name):
         exists = False
         prev_rc = []
         for rc in parsed_json['items']:
-            if 'name' in rc['metadata']['labels'] and name == rc['metadata']['labels']['name']:
+            if('name' in rc['metadata']['labels'] and name == rc['metadata']['labels']['name'] and
+               'type' in rc['spec']['selector'] and app_type == rc['spec']['selector']['type']):
                 exists = True
                 prev_rc = rc
                 break
@@ -154,7 +158,8 @@ def _get_rc_(self, name):
 
     def deploy(self, name, image, command, **kwargs):
         app_name = kwargs.get('aname', {})
-        old_rc = self._get_old_rc(app_name)
+        app_type = name.split(".")[1]
+        old_rc = self._get_old_rc(app_name, app_type)
         new_rc = self._create_rc(name, image, command, **kwargs)
         desired = int(old_rc["spec"]["replicas"])
         old_rc_name = old_rc["metadata"]["name"]
@@ -257,12 +262,12 @@ def _scale_app(self, name, num):
         self._scale_rc(js_template)
 
     def scale(self, name, image, command, **kwargs):
-        rc_name = name.split(".")[0]
+        rc_name = name.replace(".", "-")
         rc_name = rc_name.replace("_", "-")
         if not 200 <= self._get_rc_status(rc_name) <= 299:
             self.create(name, image, command, **kwargs)
             return
-        name = name.split(".")[0]
+        name = name.replace(".", "-")
         name = name.replace("_", "-")
         num = kwargs.get('num', {})
         js_template = self._get_rc_(name)
@@ -279,7 +284,7 @@ def _create_rc(self, name, image, command, **kwargs):
         app_name = kwargs.get('aname', {})
         app_type = name.split(".")[1]
         container_name = app_name+"-"+app_type
-        name = name.split(".")[0]
+        name = name.replace(".", "-")
         name = name.replace("_", "-")
         args = command.split()
 
@@ -292,6 +297,7 @@ def _create_rc(self, name, image, command, **kwargs):
         l["image"] = self.registry+"/"+image
         l['num'] = num
         l['containername'] = container_name
+        l['type'] = app_type
         template = string.Template(RC_TEMPLATE).substitute(l)
         js_template = json.loads(template)
         containers = js_template["spec"]["template"]["spec"]["containers"]
@@ -338,17 +344,36 @@ def _create_rc(self, name, image, command, **kwargs):
 
     def create(self, name, image, command, **kwargs):
         self._create_rc(name, image, command, **kwargs)
-        name = name.split(".")[0]
+        app_type = name.split(".")[1]
+        name = name.replace(".", "-")
         name = name.replace("_", "-")
         app_name = kwargs.get('aname', {})
         try:
-            self._create_service(name, app_name)
+            self._create_service(name, app_name, app_type)
         except Exception as e:
+            self._scale_app(name, 0)
             self._delete_rc(name)
             err = '{} (create): {}'.format(name, e)
             raise RuntimeError(err)
 
-    def _create_service(self, name, app_name):
+    def _get_service(self, name):
+        con_get = httplib.HTTPConnection(self.target+":"+self.port)
+        con_get.request('GET', '/api/'+self.apiversion+'/namespaces/default/services/'+name)
+        resp = con_get.getresponse()
+        reason = resp.reason
+        status = resp.status
+        data = resp.read()
+        con_get.close()
+        if not 200 <= status <= 299:
+            errmsg = "Failed to get Service: {} {} - {}".format(
+                status, reason, data)
+            raise RuntimeError(errmsg)
+        return (status, data, reason)
+
+    def _create_service(self, name, app_name, app_type):
+        random.seed(app_name)
+        app_id = random.randint(1, 100000)
+        appname = "app-"+str(app_id)
         actual_pod = {}
         for _ in xrange(300):
             status, data, reason = self._get_pods()
@@ -370,9 +395,8 @@ def _create_service(self, name, app_name):
         l["version"] = self.apiversion
         l["label"] = app_name
         l["port"] = port
-        random.seed(app_name)
-        app_id = random.randint(1, 100000)
-        l["name"] = "app-"+str(app_id)
+        l['type'] = app_type
+        l["name"] = appname
         template = string.Template(SERVICE_TEMPLATE).substitute(l)
         headers = {'Content-Type': 'application/json'}
         conn_serv = httplib.HTTPConnection(self.target+":"+self.port)
@@ -383,7 +407,27 @@ def _create_service(self, name, app_name):
         reason = resp.reason
         status = resp.status
         conn_serv.close()
-        if not 200 <= status <= 299:
+        if status == 409:
+            status, data, reason = self._get_service(appname)
+            srv = json.loads(data)
+            if srv['spec']['selector']['type'] == 'web':
+                return
+            srv['spec']['selector']['type'] = app_type
+            srv['spec']['ports'][0]['targetPort'] = port
+            headers = {'Content-Type': 'application/json'}
+            conn_scalepod = httplib.HTTPConnection(self.target+":"+self.port)
+            conn_scalepod.request('PUT', '/api/'+self.apiversion+'/namespaces/default/' +
+                                  'services/'+appname, headers=headers, body=json.dumps(srv))
+            resp = conn_scalepod.getresponse()
+            data = resp.read()
+            reason = resp.reason
+            status = resp.status
+            conn_scalepod.close()
+            if not 200 <= status <= 299:
+                errmsg = "Failed to update the Service:{} {} {} - {}".format(
+                    name, status, reason, data)
+                raise RuntimeError(errmsg)
+        elif not 200 <= status <= 299:
             errmsg = "Failed to create Service:{} {} {} - {}".format(
                      name, status, reason, data)
             raise RuntimeError(errmsg)
@@ -417,9 +461,10 @@ def _delete_rc(self, name):
 
     def destroy(self, name):
         """
-        Destroy a container
+        Destroy a the app
         """
-        name = name.split(".")[0]
+        name = name.split(".")
+        name = name[0]+'-'+name[1]
         name = name.replace("_", "-")
         appname = ''
         try:
@@ -454,7 +499,7 @@ def destroy(self, name):
         status = resp.status
         data = resp.read()
         con_serv.close()
-        if not 200 <= status <= 299:
+        if status != 404 and not 200 <= status <= 299:
             errmsg = "Failed to delete service:{} {} {} - {}".format(
                 name, status, reason, data)
             raise RuntimeError(errmsg)
@@ -529,7 +574,7 @@ def _pod_log(self, name):
         return (status, data, reason)
 
     def logs(self, name):
-        name = name.split(".")[0]
+        name = name.replace(".", "-")
         name = name.replace("_", "-")
         status, data, reason = self._get_pods()
         parsed_json = json.loads(data)
@@ -544,7 +589,7 @@ def run(self, name, image, entrypoint, command):
         """
         Run a one-off command
         """
-        name = name.split(".")[0]
+        name = name.replace(".", "-")
         name = name.replace("_", "-")
         l = {}
         l["id"] = name
@@ -603,7 +648,8 @@ def run(self, name, image, entrypoint, command):
 
     def _get_pod_state(self, name):
         try:
-            name = name.split(".")[0]
+            name = name.split(".")
+            name = name[0]+'-'+name[1]
             name = name.replace("_", "-")
             for _ in xrange(120):
                 status, data, reason = self._get_pods()

deisctl/units/deis-kube-apiserver.service

@@ -19,10 +19,11 @@ ExecStart=/bin/bash -c "/opt/bin/kube-apiserver \
   --kubelet_https=true \
   --secure_port=6443 \
   --service-cluster-ip-range=10.100.0.0/16 \
-  --etcd_servers=http://$(curl -s 127.0.0.1:4001/v2/machines | sed 's/ //g') \
+  --etcd_servers=$(curl -s 127.0.0.1:4001/v2/machines | sed 's/ //g') \
   --public_address_override=${COREOS_PRIVATE_IPV4} \
   --logtostderr=true \
   --runtime_config=api/v1"
+ExecStartPost=/bin/bash -c "fleetctl stop deis-kube-scheduler deis-kube-controller-manager deis-kube-kubelet deis-kube-proxy; sleep 2; fleetctl start deis-kube-scheduler deis-kube-controller-manager deis-kube-kubelet deis-kube-proxy"
 Restart=always
 RestartSec=10
 [Install]

deisctl/units/deis-kube-controller-manager.service

@@ -6,6 +6,7 @@ After=deis-kube-apiserver.service
 [Service]
 EnvironmentFile=/etc/environment
 ExecStartPre=/bin/bash -c "/opt/bin/download-k8s-binary kube-controller-manager"
+ExecStartPre=/bin/bash -c "/opt/bin/wupiao $(/usr/bin/etcdctl get /deis/scheduler/k8s/master):8080"
 ExecStart=/opt/bin/kube-controller-manager \
   --service_account_private_key_file=/opt/bin/kube-serviceaccount.key \
   --master=127.0.0.1:8080 \

deisctl/units/deis-kube-kubelet.service

@@ -4,6 +4,7 @@ Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 [Service]
 EnvironmentFile=/etc/environment
 ExecStartPre=/bin/bash -c "/opt/bin/download-k8s-binary kubelet"
+ExecStartPre=/bin/bash -c "/opt/bin/wupiao $(/usr/bin/etcdctl get /deis/scheduler/k8s/master):8080"
 ExecStartPre=/usr/bin/mkdir -p /opt/kubernetes/manifests/
 ExecStart=/bin/bash -c '/opt/bin/kubelet --address=0.0.0.0 --port=10250 --hostname_override=$COREOS_PRIVATE_IPV4 --api_servers=`/usr/bin/etcdctl get /deis/scheduler/k8s/master`:8080 --allow_privileged=true --cluster_dns=10.100.0.10 --cluster_domain=cluster.local --logtostderr=true --config=/opt/kubernetes/manifests/ --healthz_bind_address=0.0.0.0 --healthz_port=10248'
 Restart=always

deisctl/units/deis-kube-proxy.service

@@ -4,6 +4,7 @@ Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 [Service]
 EnvironmentFile=/etc/environment
 ExecStartPre=/bin/bash -c "/opt/bin/download-k8s-binary kube-proxy"
+ExecStartPre=/bin/bash -c "/opt/bin/wupiao $(/usr/bin/etcdctl get /deis/scheduler/k8s/master):8080"
 ExecStart=/bin/bash -c '/opt/bin/kube-proxy --master=`/usr/bin/etcdctl get /deis/scheduler/k8s/master`:8080 --logtostderr=true'
 Restart=always
 RestartSec=10

deisctl/units/deis-kube-scheduler.service

@@ -6,8 +6,10 @@ After=deis-kube-apiserver.service
 [Service]
 EnvironmentFile=/etc/environment
 ExecStartPre=/bin/bash -c "/opt/bin/download-k8s-binary kube-scheduler"
+ExecStartPre=/bin/bash -c "/opt/bin/wupiao $(/usr/bin/etcdctl get /deis/scheduler/k8s/master):8080"
 ExecStart=/opt/bin/kube-scheduler \
-  --master=127.0.0.1:8080
+  --master=127.0.0.1:8080 \
+  --policy-config-file=/opt/bin/scheduler-policy.json
 Restart=always
 RestartSec=10
 [X-Fleet]