Merge pull request #70 from notmaxx/feature_timeouts

feat(controller): support timeouts per deployment

Author: Cryptophobia

rootfs/api/migrations/0027_config_termination_grace_period.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.4 on 2018-04-24 01:41
+from __future__ import unicode_literals
+
+from django.db import migrations
+import jsonfield.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0026_release_exception'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='config',
+            name='termination_grace_period',
+            field=jsonfield.fields.JSONField(blank=True, default={}),
+        ),
+    ]

rootfs/api/models/app.py

@@ -1107,6 +1107,7 @@ def _gather_app_settings(self, release, app_settings, process_type, replicas):
             'deployment_revision_history_limit': deployment_history,
             'release_summary': release.summary,
             'pod_termination_grace_period_seconds': pod_termination_grace_period_seconds,
+            'pod_termination_grace_period_each': config.termination_grace_period,
             'image_pull_secret_name': image_pull_secret_name,
             'image_pull_policy': image_pull_policy
         }

rootfs/api/models/config.py

@@ -24,6 +24,7 @@ class Config(UuidAuditedModel):
     tags = JSONField(default={}, blank=True)
     registry = JSONField(default={}, blank=True)
     healthcheck = JSONField(default={}, blank=True)
+    termination_grace_period = JSONField(default={}, blank=True)
 
     class Meta:
         get_latest_by = 'created'
@@ -165,7 +166,8 @@ def save(self, **kwargs):
                 previous_config = self.app.config_set.latest()
 
             for attr in ['cpu', 'memory', 'tags', 'registry', 'values',
-                         'lifecycle_post_start', 'lifecycle_pre_stop']:
+                         'lifecycle_post_start', 'lifecycle_pre_stop',
+                         'termination_grace_period']:
                 data = getattr(previous_config, attr, {}).copy()
                 new_data = getattr(self, attr, {}).copy()
 

rootfs/api/models/release.py

@@ -459,6 +459,15 @@ def save(self, *args, **kwargs):  # noqa
                 if changes:
                     if self.summary:
                         self.summary += ' and '
+
+                # if the timeouts changed, log the dict diff
+                changes = []
+                old_timeout = old_config.termination_grace_period if old_config else {}
+                diff = dict_diff(self.config.termination_grace_period, old_timeout)
+                if diff.get('added') or diff.get('changed') or diff.get('deleted'):
+                    changes.append('termination_grace_period')
+                if changes:
+                    changes = 'changed timeouts for '+', '.join(changes)
                     self.summary += "{} {}".format(self.config.owner, changes)
 
                 # if the tags changed, log the dict diff

rootfs/api/serializers.py

@@ -26,6 +26,7 @@
     r'^(?P<cpu>(([-+]?[0-9]*\.?[0-9]+[m]?)(/([-+]?[0-9]*\.?[0-9]+[m]?))?))$')
 TAGVAL_MATCH = re.compile(r'^(?:[a-zA-Z\d][-\.\w]{0,61})?[a-zA-Z\d]$')
 CONFIGKEY_MATCH = re.compile(r'^[a-z_]+[a-z0-9_]*$', re.IGNORECASE)
+TERMINATION_GRACE_PERIOD_MATCH = re.compile(r'^[0-9]*$')
 PROBE_SCHEMA = {
     "$schema": "http://json-schema.org/schema#",
 
@@ -218,6 +219,7 @@ class ConfigSerializer(serializers.ModelSerializer):
     registry = JSONFieldSerializer(required=False, binary=True)
     healthcheck = JSONFieldSerializer(convert_to_str=False, required=False, binary=True)
     routable = serializers.BooleanField(required=False)
+    termination_grace_period = JSONFieldSerializer(required=False, binary=True)
 
     class Meta:
         """Metadata options for a :class:`ConfigSerializer`."""
@@ -299,6 +301,21 @@ def validate_cpu(self, data):
 
         return data
 
+    def validate_termination_grace_period(self, data):
+        for key, value in data.items():
+            if value is None:  # use NoneType to unset an item
+                continue
+
+            if not re.match(PROCTYPE_MATCH, key):
+                raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
+
+            timeout = re.match(TERMINATION_GRACE_PERIOD_MATCH, str(value))
+            if not timeout:
+                raise serializers.ValidationError(
+                    "Termination Grace Period format: <value>, where value must be a numeric")
+
+        return data
+
     def validate_tags(self, data):
         for key, value in data.items():
             if value is None:  # use NoneType to unset an item

rootfs/api/tests/test_config.py

@@ -166,7 +166,7 @@ def test_response_data(self, mock_requests):
         for key in response.data:
             self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'values', 'memory',
                                 'cpu', 'tags', 'registry', 'healthcheck', 'lifecycle_post_start',
-                                'lifecycle_pre_stop'])
+                                'lifecycle_pre_stop', 'termination_grace_period'])
         expected = {
             'owner': self.user.username,
             'app': app_id,
@@ -190,7 +190,7 @@ def test_response_data_types_converted(self, mock_requests):
         for key in response.data:
             self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'values', 'memory',
                                 'cpu', 'tags', 'registry', 'healthcheck', 'lifecycle_post_start',
-                                'lifecycle_pre_stop'])
+                                'lifecycle_pre_stop', 'termination_grace_period'])
         expected = {
             'owner': self.user.username,
             'app': app_id,

rootfs/scheduler/resources/pod.py

@@ -130,7 +130,7 @@ def manifest(self, namespace, name, image, **kwargs):
         spec['nodeSelector'] = kwargs.get('tags', {})
 
         # How long until a pod is forcefully terminated. 30 is kubernetes default
-        spec['terminationGracePeriodSeconds'] = kwargs.get('pod_termination_grace_period_seconds', 30)  # noqa
+        spec['terminationGracePeriodSeconds'] = self._get_termination_grace_period(kwargs)  # noqa
 
         # Check if it is a slug builder image.
         if build_type == "buildpack":
@@ -255,6 +255,14 @@ def _set_resources(self, container, kwargs):
             if resources:
                 container["resources"] = dict(resources)
 
+    def _get_termination_grace_period(self, kwargs):
+        """ return termination grace period """
+        app_type = kwargs.get("app_type")
+        timeout_global = kwargs.get('pod_termination_grace_period_seconds', 30)
+        timeout_local = kwargs.get("pod_termination_grace_period_each", {}).get(app_type)
+
+        return timeout_global if timeout_local is None else int(timeout_local)
+
     def _format_memory(self, mem):
         """ Format memory limit value """
         if mem[-2:-1].isalpha() and mem[-1].isalpha():