feat(scheduler): add support for set based requirement filtering via the kubernetes API

Makes it possible to use a notin() and in() filter.

This can be achieved by using this syntax: version__notin(v3, v4, v5) or type__in(cmd, web, public)

Author: helgi

rootfs/api/models/release.py

@@ -258,9 +258,7 @@ def cleanup_old(self):
         )
 
         # Cleanup controllers
-        labels = {
-            'heritage': 'deis'
-        }
+        labels = {'heritage': 'deis'}
         controller_removal = []
         controllers = self._scheduler.get_rcs(self.app.id, labels=labels).json()
         for controller in controllers['items']:
@@ -299,9 +297,7 @@ def cleanup_old(self):
             self._scheduler.delete_secret(self.app.id, secret['metadata']['name'])
 
         # Remove stray pods
-        labels = {
-            'heritage': 'deis'
-        }
+        labels = {'heritage': 'deis'}
         pods = self._scheduler.get_pods(self.app.id, labels=labels).json()
         for pod in pods['items']:
             if self._scheduler.pod_deleted(pod):

rootfs/scheduler/__init__.py

@@ -605,9 +605,24 @@ def _selectors(self, **kwargs):
         # labels and fields are encoded slightly differently than python-requests can do
         labels = kwargs.get('labels', {})
         if labels:
-            # http://kubernetes.io/v1.1/docs/user-guide/labels.html#list-and-watch-filtering
-            labels = ['{}={}'.format(key, value) for key, value in labels.items()]
-            query['labelSelector'] = ','.join(labels)
+            selectors = []
+            for key, value in labels.items():
+                # http://kubernetes.io/docs/user-guide/labels/#set-based-requirement
+                if '__notin' in key:
+                    key = key.replace('__notin', '')
+                    selectors.append('{} notin({})'.format(key, ','.join(value)))
+                # list is automagically a in()
+                elif '__in' in key or isinstance(value, list):
+                    key = key.replace('__in', '')
+                    selectors.append('{} in({})'.format(key, ','.join(value)))
+                elif value is None:
+                    # allowing a check if a label exists (or not) without caring about value
+                    selectors.append(key)
+                # http://kubernetes.io/docs/user-guide/labels/#equality-based-requirement
+                elif isinstance(value, str):
+                    selectors.append('{}={}'.format(key, value))
+
+            query['labelSelector'] = ','.join(selectors)
 
         fields = kwargs.get('fields', {})
         if fields:

rootfs/scheduler/mock.py

@@ -4,6 +4,7 @@
 from urllib.parse import urlparse, parse_qs
 import string
 import random
+import re
 import time
 
 from . import KubeHTTPClient, KubeHTTPException
@@ -275,8 +276,9 @@ def upsert_pods(controller, url):
 
 def filter_data(filters, path):
     data = []
-    for item in cache.get(path, []):
-        item = cache.get(item)
+    rows = cache.get(path, [])
+    for row in rows:
+        item = cache.get(row)
         if not item:
             # item is broken
             continue
@@ -288,7 +290,25 @@ def filter_data(filters, path):
         # Do extra filtering based on labelSelector
         add = True
         for label, value in filters['labels'].items():
-            if (
+            # set based filter
+            if '__' in label:
+                label, matcher = label.split('__')
+                if matcher == 'in':
+                    if (
+                        label not in item['metadata']['labels'] or
+                        item['metadata']['labels'][label] not in value
+                    ):
+                        add = False
+                        continue
+                elif matcher == 'notin':
+                    if (
+                        label not in item['metadata']['labels'] or
+                        item['metadata']['labels'][label] in value
+                    ):
+                        add = False
+                        continue
+
+            elif (
                 label not in item['metadata']['labels'] or
                 item['metadata']['labels'][label] != value
             ):
@@ -325,12 +345,29 @@ def fetch_all(request, context):
 def prepare_query_filters(query):
     filters = {'labels': {}, 'fields': {}}
     if query:
+        # set based regex - does not support only field
+        labelRegex = re.compile('^(?P<label>.*) (?P<matcher>notin|in)\s?\((?P<values>.*)\)$')
+
         queries = parse_qs(query)
         if 'labelSelector' in queries:
             for items in queries['labelSelector']:
-                for item in items.split(','):
-                    key, value = item.split('=')
-                    filters['labels'][key] = value
+                # split on , but not inside ()
+                r = re.compile(r'(?:[^,(]|\([^)]*\))+')
+                for item in r.findall(items):
+                    if '=' in item:
+                        # equal based requirement
+                        key, value = item.split('=')
+                        filters['labels'][key] = value
+                    else:
+                        # set based requirement
+                        matches = labelRegex.match(item)
+                        if matches is None:
+                            continue
+
+                        # split and strip spaces
+                        values = [x.strip() for x in matches.group('values').split(',')]
+                        key = matches.group('label') + '__' + matches.group('matcher')
+                        filters['labels'][key] = values
 
         if 'fieldSelector' in queries:
             for items in queries['fieldSelector']: