Merge pull request #95 from smothiki/newbuild

feat(workflow): identify app type and move envs to rc templates

Author: Sivaram Mothiki

rootfs/api/models.py

@@ -316,6 +316,7 @@ def _scale_containers(self, scale_types, to_remove):
             kwargs = {'memory': release.config.memory,
                       'cpu': release.config.cpu,
                       'tags': release.config.tags,
+                      'envs': release.config.values,
                       'version': version,
                       'aname': self.id,
                       'num': scale_types[scale_type]}
@@ -472,6 +473,7 @@ def _deploy_app(self, scale_types, release, existing):
             kwargs = {'memory': release.config.memory,
                       'cpu': release.config.cpu,
                       'tags': release.config.tags,
+                      'envs': release.config.values,
                       'aname': self.id,
                       'num': 0,
                       'version': version}
@@ -615,7 +617,8 @@ def create(self):
         image = self.release.image
         kwargs = {'memory': self.release.config.memory,
                   'cpu': self.release.config.cpu,
-                  'tags': self.release.config.tags}
+                  'tags': self.release.config.tags,
+                  'envs': self.release.config.values}
         try:
             self._scheduler.create(
                 name=self.job_id,
@@ -831,7 +834,9 @@ def __str__(self):
 
     @property
     def image(self):
-        return '{}:v{}'.format(self.app.id, str(self.version))
+        if not self.build.dockerfile and not self.build.sha:
+            return '{}:v{}'.format(self.app.id, str(self.version))
+        return '{}:git-{}'.format(self.app.id, str(self.build.sha))
 
     def new(self, user, config, build, summary=None, source_version='latest'):
         """
@@ -863,7 +868,8 @@ def publish(self, source_version='latest'):
             source_image = "{}:{}".format(source_image, source_tag)
         # If the build has a SHA, assume it's from deis-builder and in the deis-registry already
         deis_registry = bool(self.build.sha)
-        publish_release(source_image, self.config.values, self.image, deis_registry)
+        if not self.build.dockerfile and not self.build.sha:
+            publish_release(source_image, self.config.values, self.image, deis_registry)
 
     def previous(self):
         """

rootfs/api/tests/test_release.py

@@ -150,13 +150,14 @@ def test_release_rollback(self):
             url, json.dumps(body), content_type='application/json',
             HTTP_AUTHORIZATION='token {}'.format(self.token))
         self.assertEqual(response.status_code, 201)
+        #  todo : edge case that fails becasue version 1 has no build object.
         # update the build to roll a new release
-        url = '/v2/apps/{app_id}/builds'.format(**locals())
-        body = {'image': 'autotest/example'}
-        response = self.client.post(
-            url, json.dumps(body), content_type='application/json',
-            HTTP_AUTHORIZATION='token {}'.format(self.token))
-        self.assertEqual(response.status_code, 201)
+        # url = '/v2/apps/{app_id}/builds'.format(**locals())
+        # body = {'image': 'autotest/example'}
+        # response = self.client.post(
+        #     url, json.dumps(body), content_type='application/json',
+        #     HTTP_AUTHORIZATION='token {}'.format(self.token))
+        # self.assertEqual(response.status_code, 201)
         # rollback and check to see that a 4th release was created
         # with the build and config of release #2
         url = "/v2/apps/{app_id}/releases/rollback/".format(**locals())
@@ -167,22 +168,22 @@ def test_release_rollback(self):
         response = self.client.get(url, content_type='application/json',
                                    HTTP_AUTHORIZATION='token {}'.format(self.token))
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(response.data['count'], 4)
-        url = '/v2/apps/{app_id}/releases/v2'.format(**locals())
+        self.assertEqual(response.data['count'], 3)
+        url = '/v2/apps/{app_id}/releases/v1'.format(**locals())
         response = self.client.get(url, content_type='application/json',
                                    HTTP_AUTHORIZATION='token {}'.format(self.token))
         self.assertEqual(response.status_code, 200)
-        release2 = response.data
-        self.assertEquals(release2['version'], 2)
-        url = '/v2/apps/{app_id}/releases/v4'.format(**locals())
+        release1 = response.data
+        self.assertEquals(release1['version'], 1)
+        url = '/v2/apps/{app_id}/releases/v3'.format(**locals())
         response = self.client.get(url, content_type='application/json',
                                    HTTP_AUTHORIZATION='token {}'.format(self.token))
         self.assertEqual(response.status_code, 200)
-        release4 = response.data
-        self.assertEquals(release4['version'], 4)
-        self.assertNotEqual(release2['uuid'], release4['uuid'])
-        self.assertEqual(release2['build'], release4['build'])
-        self.assertEqual(release2['config'], release4['config'])
+        release3 = response.data
+        self.assertEquals(release3['version'], 3)
+        self.assertNotEqual(release1['uuid'], release3['uuid'])
+        self.assertEqual(release1['build'], release3['build'])
+        self.assertEqual(release1['config'], release3['config'])
         # rollback explicitly to release #1 and check that a 5th release
         # was created with the build and config of release #1
         url = "/v2/apps/{app_id}/releases/rollback/".format(**locals())
@@ -195,27 +196,27 @@ def test_release_rollback(self):
         response = self.client.get(url, content_type='application/json',
                                    HTTP_AUTHORIZATION='token {}'.format(self.token))
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(response.data['count'], 5)
+        self.assertEqual(response.data['count'], 4)
         url = '/v2/apps/{app_id}/releases/v1'.format(**locals())
         response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token))
         self.assertEqual(response.status_code, 200)
         release1 = response.data
-        url = '/v2/apps/{app_id}/releases/v5'.format(**locals())
+        url = '/v2/apps/{app_id}/releases/v4'.format(**locals())
         response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token))
         self.assertEqual(response.status_code, 200)
-        release5 = response.data
-        self.assertEqual(release5['version'], 5)
-        self.assertNotEqual(release1['uuid'], release5['uuid'])
-        self.assertEqual(release1['build'], release5['build'])
-        self.assertEqual(release1['config'], release5['config'])
+        release4 = response.data
+        self.assertEqual(release4['version'], 4)
+        self.assertNotEqual(release1['uuid'], release4['uuid'])
+        self.assertEqual(release1['build'], release4['build'])
+        self.assertEqual(release1['config'], release4['config'])
         # check to see that the current config is actually the initial one
         url = "/v2/apps/{app_id}/config".format(**locals())
         response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token))
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.data['values'], {})
-        # rollback to #3 and see that it has the correct config
+        # rollback to #2 and see that it has the correct config
         url = "/v2/apps/{app_id}/releases/rollback/".format(**locals())
-        body = {'version': 3}
+        body = {'version': 2}
         response = self.client.post(
             url, json.dumps(body), content_type='application/json',
             HTTP_AUTHORIZATION='token {}'.format(self.token))

rootfs/deis/settings.py

@@ -311,6 +311,12 @@
 REGISTRY_PORT = os.environ.get('DEIS_REGISTRY_SERVICE_PORT', 5000)
 REGISTRY_URL = '{}:{}'.format(REGISTRY_HOST, REGISTRY_PORT)
 
+BUILDER_HOST = os.environ.get('DEIS_BUILDER_SERVICE_HOST', '127.0.0.1')
+MINIO_HOST = os.environ.get('DEIS_MINIO_SERVICE_HOST', BUILDER_HOST)
+MINIO_PORT = os.environ.get('DEIS_MINIO_SERVICE_PORT', 3000)
+S3EP_HOST = os.environ.get('DEIS_OUTSIDE_STORAGE_HOST', MINIO_HOST)
+S3EP_PORT = os.environ.get('DEIS_OUTSIDE_STORAGE_PORT', MINIO_PORT)
+S3EP = '{}:{}'.format(S3EP_HOST, S3EP_PORT)
 # logger settings
 LOGGER_HOST = os.environ.get('DEIS_LOGGER_SERVICE_HOST', '127.0.0.1')
 LOGGER_PORT = os.environ.get('DEIS_LOGGER_SERVICE_PORT', 8088)

rootfs/scheduler/k8s.py

@@ -36,7 +36,7 @@
 }
 """
 
-RC_TEMPLATE = """\
+RCD_TEMPLATE = """\
 {
   "kind": "ReplicationController",
   "apiVersion": "$version",
@@ -68,7 +68,80 @@
         "containers": [
           {
             "name": "$containername",
-            "image": "$image"
+            "image": "$image",
+            "env": [
+            {
+                "name":"DEIS_APP",
+                "value":"$id"
+            },
+            {
+                "name":"DEIS_RELEASE",
+                "value":"$appversion"
+            }
+            ]
+          }
+        ]
+      }
+    }
+  }
+}
+"""
+
+RCB_TEMPLATE = """\
+{
+  "kind": "ReplicationController",
+  "apiVersion": "$version",
+  "metadata": {
+    "name": "$name",
+    "labels": {
+      "app": "$id",
+      "heritage": "deis"
+    }
+  },
+  "spec": {
+    "replicas": $num,
+    "selector": {
+      "app": "$id",
+      "version": "$appversion",
+      "type": "$type",
+      "heritage": "deis"
+    },
+    "template": {
+      "metadata": {
+        "labels": {
+          "app": "$id",
+          "version": "$appversion",
+          "type": "$type",
+          "heritage": "deis"
+        }
+      },
+      "spec": {
+        "containers": [
+          {
+            "name": "$containername",
+            "image": "quay.io/deisci/slugrunner:v2-alpha",
+            "env": [
+            {
+                "name":"PORT",
+                "value":"5000"
+            },
+            {
+                "name":"SLUG_URL",
+                "value":"$image"
+            },
+            {
+                "name":"DEBUG",
+                "value":"1"
+            },
+            {
+                "name":"DEIS_APP",
+                "value":"$id"
+            },
+            {
+                "name":"DEIS_RELEASE",
+                "value":"$appversion"
+            }
+            ]
           }
         ]
       }
@@ -156,7 +229,7 @@ def _get_old_rc(self, name, app_type):
         exists = False
         prev_rc = []
         for rc in resp.json()['items']:
-            if('name' in rc['metadata']['labels'] and name == rc['metadata']['labels']['name'] and
+            if('app' in rc['spec']['selector'] and name == rc['metadata']['labels']['app'] and
                'type' in rc['spec']['selector'] and app_type == rc['spec']['selector']['type']):
                 exists = True
                 prev_rc = rc
@@ -185,6 +258,7 @@ def deploy(self, name, image, command, **kwargs):
         app_type = name.split(".")[1]
         old_rc = self._get_old_rc(app_name, app_type)
         new_rc = self._create_rc(name, image, command, **kwargs)
+        old_temp_rc = old_rc
         if old_rc:
             desired = int(old_rc["spec"]["replicas"])
             old_rc_name = old_rc["metadata"]["name"]
@@ -196,8 +270,8 @@ def deploy(self, name, image, command, **kwargs):
             count = 1
             while desired >= count:
                 new_rc = self._scale_app(new_rc_name, count, app_name)
-                if old_rc:
-                    old_rc = self._scale_app(old_rc_name, desired-count, app_name)
+                if old_temp_rc:
+                    old_temp_rc = self._scale_app(old_rc_name, desired-count, app_name)
                 count += 1
         except Exception as e:
             self._scale_app(new_rc["metadata"]["name"], 0, app_name)
@@ -207,7 +281,7 @@ def deploy(self, name, image, command, **kwargs):
 
             raise RuntimeError('{} (deploy): {}'.format(name, e))
         if old_rc:
-            self._delete_rc(old_rc_name, app_name)
+            self._delete_rc(app_name, old_rc_name)
 
     def _get_events(self, namespace):
         url = self._api("/namespaces/{}/events", namespace)
@@ -305,6 +379,25 @@ def scale(self, name, image, command, **kwargs):
             self._scale_app(name, old_replicas, app_name)
             raise RuntimeError('{} (Scale): {}'.format(name, e))
 
+    def _create_namespace(self, app_name):
+        url = self._api("/namespaces")
+        data = {
+            "kind": "Namespace",
+            "apiVersion": self.apiversion,
+            "metadata": {
+                "name": app_name
+                }
+            }
+        resp = self.session.post(url, json=data)
+        if not resp.status_code == 201:
+            error(resp, "create Namespace {}".format(app_name))
+
+    def _check_status(self, resp, app_name):
+        if resp.status_code == 404:
+            self._create_namespace(app_name)
+        elif resp.status_code != 200:
+            error(resp, "locate Namespace {}".format(app_name))
+
     def _create_rc(self, name, image, command, **kwargs):
         container_fullname = name
         app_name = kwargs.get('aname', {})
@@ -316,42 +409,36 @@ def _create_rc(self, name, image, command, **kwargs):
         # First ensure that the namespace was created
         url = self._api("/namespaces/{}", app_name)
         resp = self.session.get(url)
-        if resp.status_code == 404:
-            url = self._api("/namespaces")
-            data = {
-                "kind": "Namespace",
-                "apiVersion": self.apiversion,
-                "metadata": {
-                    "name": app_name
-                }
-            }
-
-            resp = self.session.post(url, json=data)
-            if not resp.status_code == 201:
-                error(resp, "create Namespace {}".format(app_name))
-        elif resp.status_code != 200:
-            error(resp, "locate Namespace {}".format(app_name))
-
+        self._check_status(resp, app_name)
         num = kwargs.get('num', {})
+        imgurl = self.registry + "/" + image
+        TEMPLATE = RCD_TEMPLATE
+        shalen = len(image[image.index(":")+5:])
+        if image[image.index(":")+1:image.index(":")+4] == "git" and shalen == 8:
+            imgurl = "http://"+settings.S3EP+"/git/home/"+image+"/slug"
+            TEMPLATE = RCB_TEMPLATE
         l = {
             "name": name,
             "id": app_name,
             "appversion": kwargs.get("version", {}),
             "version": self.apiversion,
-            "image": self.registry + "/" + image,
+            "image": imgurl,
             "num": kwargs.get("num", {}),
             "containername": container_name,
             "type": app_type,
         }
-
-        template = string.Template(RC_TEMPLATE).substitute(l)
+        template = string.Template(TEMPLATE).substitute(l)
         js_template = json.loads(template)
         containers = js_template["spec"]["template"]["spec"]["containers"]
         containers[0]['args'] = args
         loc = locals().copy()
         loc.update(re.match(MATCH, container_fullname).groupdict())
         mem = kwargs.get('memory', {}).get(loc['c_type'])
         cpu = kwargs.get('cpu', {}).get(loc['c_type'])
+        env = kwargs.get('envs', {})
+        if env:
+            for k, v in env.iteritems():
+                containers[0]["env"].append({"name": k, "value": v})
         if mem or cpu:
             containers[0]["resources"] = {"limits": {}}
 
@@ -365,13 +452,11 @@ def _create_rc(self, name, image, command, **kwargs):
         if cpu:
             cpu = float(cpu)/1024
             containers[0]["resources"]["limits"]["cpu"] = cpu
-
         url = self._api("/namespaces/{}/replicationcontrollers", app_name)
         resp = self.session.post(url, json=js_template)
         if unhealthy(resp.status_code):
             error(resp, 'create ReplicationController "{}" in Namespace "{}"',
                   name, app_name)
-
         create = False
         for _ in xrange(30):
             if not create and self._get_rc_status(name, app_name) == 404:
@@ -483,7 +568,7 @@ def stop(self, name):
         """Stop a container."""
         pass
 
-    def _delete_rc(self, name, namespace):
+    def _delete_rc(self, namespace, name):
         url = self._api("/namespaces/{}/replicationcontrollers/{}", namespace, name)
         resp = self.session.delete(url)
         if unhealthy(resp.status_code):