fix(scheduler): create application config (env secrets) outside of deploy / scale to reduce k8s thrasing

helgi · helgi · commit 855bb59db379 · 2016-08-05T12:15:18.000-07:00
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
@@ -389,7 +389,9 @@ def scale(self, user, structure):  # noqa
 
     def _scale_pods(self, scale_types):
         release = self.release_set.latest()
-        envs = release.config.values
+        version = "v{}".format(release.version)
+        image = release.image
+        envs = self._build_env_vars(release.build.type, version, image, release.config.values)
 
         # see if the app config has deploy batch preference, otherwise use global
         batches = release.config.values.get('DEIS_DEPLOY_BATCHES', settings.DEIS_DEPLOY_BATCHES)
@@ -413,7 +415,7 @@ def _scale_pods(self, scale_types):
                 'tags': release.config.tags,
                 'envs': envs,
                 'registry': release.config.registry,
-                'version': "v{}".format(release.version),
+                'version': version,
                 'replicas': replicas,
                 'app_type': scale_type,
                 'build_type': release.build.type,
@@ -429,14 +431,17 @@ def _scale_pods(self, scale_types):
                     self._scheduler.scale,
                     namespace=self.id,
                     name=self._get_job_id(scale_type),
-                    image=release.image,
+                    image=image,
                     entrypoint=self._get_entrypoint(scale_type),
                     command=self._get_command(scale_type),
                     **kwargs
                 )
             )
 
         try:
+            # create the application config in k8s (secret in this case) for all deploy objects
+            self._scheduler.set_application_config(self.id, envs, version)
+
             async_run(tasks)
         except Exception as e:
             err = '(scale): {}'.format(e)
@@ -469,7 +474,10 @@ def deploy(self, release, force_deploy=False):
 
         # deploy application to k8s. Also handles initial scaling
         deploys = {}
-        envs = release.config.values
+        image = release.image
+        version = "v{}".format(release.version)
+        envs = self._build_env_vars(release.build.type, version, image, release.config.values)
+
         for scale_type, replicas in self.structure.items():
             # only web / cmd are routable
             # http://docs.deis.io/en/latest/using_deis/process-types/#web-vs-cmd-process-types
@@ -487,7 +495,7 @@ def deploy(self, release, force_deploy=False):
                 'registry': release.config.registry,
                 # only used if there is no previous RC
                 'replicas': replicas,
-                'version': "v{}".format(release.version),
+                'version': version,
                 'app_type': scale_type,
                 'build_type': release.build.type,
                 'healthcheck': release.config.healthcheck,
@@ -509,13 +517,16 @@ def deploy(self, release, force_deploy=False):
                 raise AlreadyExists('Deployment for {} is already in progress'.format(name))
 
         try:
+            # create the application config in k8s (secret in this case) for all deploy objects
+            self._scheduler.set_application_config(self.id, envs, version)
+
             # gather all proc types to be deployed
             tasks = [
                 functools.partial(
                     self._scheduler.deploy,
                     namespace=self.id,
                     name=self._get_job_id(scale_type),
-                    image=release.image,
+                    image=image,
                     entrypoint=self._get_entrypoint(scale_type),
                     command=self._get_command(scale_type),
                     **kwargs
@@ -682,13 +693,16 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
         name = self._get_job_id(scale_type) + '-' + pod_name()
         self.log("{} on {} runs '{}'".format(user.username, name, command))
 
+        image = release.image
+        version = "v{}".format(release.version)
+        envs = self._build_env_vars(release.build.type, version, image, release.config.values)
         kwargs = {
             'memory': release.config.memory,
             'cpu': release.config.cpu,
             'tags': release.config.tags,
-            'envs': release.config.values,
+            'envs': envs,
             'registry': release.config.registry,
-            'version': "v{}".format(release.version),
+            'version': version,
             'build_type': release.build.type,
             'deploy_timeout': deploy_timeout
         }
@@ -697,7 +711,7 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
             exit_code, output = self._scheduler.run(
                 self.id,
                 name,
-                release.image,
+                image,
                 self._get_entrypoint(scale_type),
                 [command],
                 **kwargs
@@ -775,3 +789,27 @@ def _scheduler_filter(self, **kwargs):
             labels.update({'type': kwargs['type']})
 
         return labels
+
+    def _build_env_vars(self, build_type, version, image, envs):
+        """
+        Build a dict of env vars, setting default vars based on app type
+        and then combining with the user set ones
+        """
+
+        # mix in default environment information deis may require
+        default_env = {
+            'DEIS_APP': self.id,
+            'WORKFLOW_RELEASE': version
+        }
+
+        # Check if it is a slug builder image.
+        if build_type == 'buildpack':
+            # overwrite image so slugrunner image is used in the container
+            default_env['SLUG_URL'] = image
+            default_env['BUILDER_STORAGE'] = settings.APP_STORAGE
+            default_env['DEIS_MINIO_SERVICE_HOST'] = settings.MINIO_HOST
+            default_env['DEIS_MINIO_SERVICE_PORT'] = settings.MINIO_PORT
+
+        # merge envs on top of default to make envs win
+        default_env.update(envs)
+        return default_env
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
@@ -290,6 +290,11 @@
 ROUTER_HOST = os.environ.get('DEIS_ROUTER_SERVICE_HOST', '127.0.0.1')
 ROUTER_PORT = os.environ.get('DEIS_ROUTER_SERVICE_PORT', 80)
 
+# minio information
+MINIO_HOST = os.environ.get('DEIS_MINIO_SERVICE_HOST', '127.0.0.1')
+MINIO_PORT = os.environ.get('DEIS_MINIO_SERVICE_PORT', 80)
+APP_STORAGE = os.environ.get('APP_STORAGE')
+
 # check if we can register users with `deis register`
 REGISTRATION_MODE = os.environ.get('REGISTRATION_MODE', 'enabled')
 
diff --git a/rootfs/scheduler/__init__.py b/rootfs/scheduler/__init__.py
@@ -392,12 +392,6 @@ def _build_pod_manifest(self, namespace, name, image, **kwargs):
         # set the image pull policy that is associated with the application container
         kwargs['image_pull_policy'] = settings.DOCKER_BUILDER_IMAGE_PULL_POLICY
 
-        # mix in default environment information deis may require
-        default_env = {
-            'DEIS_APP': namespace,
-            'WORKFLOW_RELEASE': kwargs.get("version")
-        }
-
         # Check if it is a slug builder image.
         if build_type == "buildpack":
             # only buildpack apps need access to object storage
@@ -422,20 +416,11 @@ def _build_pod_manifest(self, namespace, name, image, **kwargs):
                 'readOnly': True
             }]
 
-            default_env['SLUG_URL'] = image
-            default_env['BUILDER_STORAGE'] = os.getenv("APP_STORAGE")
-            default_env['DEIS_MINIO_SERVICE_HOST'] = os.getenv("DEIS_MINIO_SERVICE_HOST")
-            default_env['DEIS_MINIO_SERVICE_PORT'] = os.getenv("DEIS_MINIO_SERVICE_PORT")
-
             # overwrite image so slugrunner image is used in the container
             image = settings.SLUGRUNNER_IMAGE
             # slugrunner pull policy
             kwargs['image_pull_policy'] = settings.SLUG_BUILDER_IMAGE_PULL_POLICY
 
-        envs = kwargs.get('envs', {})
-        default_env.update(envs)
-        kwargs['envs'] = default_env
-
         # create the base container
         container = {}
 
@@ -469,6 +454,8 @@ def run(self, namespace, name, image, entrypoint, command, **kwargs):
         kwargs['command'] = entrypoint
         kwargs['args'] = command
 
+        # create application config and build the pod manifest
+        self.set_application_config(namespace, kwargs.get('envs', {}), kwargs.get('version'))
         manifest = self._build_pod_manifest(namespace, name, image, **kwargs)
 
         url = self._api("/namespaces/{}/pods", namespace)
@@ -530,7 +517,7 @@ def run(self, namespace, name, image, entrypoint, command, **kwargs):
             # cleanup
             self.delete_pod(namespace, name)
 
-    def set_application_config(self, namespace, env, version):
+    def set_application_config(self, namespace, envs, version):
         # env vars are stored in secrets and mapped to env in k8s
         try:
             labels = {
@@ -540,7 +527,7 @@ def set_application_config(self, namespace, env, version):
 
             # secrets use dns labels for keys, map those properly here
             secrets_env = {}
-            for key, value in env.items():
+            for key, value in envs.items():
                 secrets_env[key.lower().replace('_', '-')] = str(value)
 
             # dictionary sorted by key
@@ -574,8 +561,6 @@ def _set_container(self, namespace, container_name, data, **kwargs):
             data['env'] = []
 
         if env:
-            self.set_application_config(namespace, env, kwargs.get('version'))
-
             # map application configuration (env secret) to env vars
             secret_name = "{}-{}-env".format(namespace, kwargs.get('version'))
             for key in env.keys():
