Merge remote-tracking branch 'upstream/master'

Author: lijianguo

charts/controller/templates/controller-deployment.yaml

@@ -128,14 +128,16 @@ spec:
               secretKeyRef:
                 name: database-creds
                 key: password
-          - name: RESERVED_NAMES
-            value: "drycc, drycc-builder, drycc-monitor-grafana"
           - name: WORKFLOW_NAMESPACE
             valueFrom:
               fieldRef:
                 fieldPath: metadata.namespace
           - name: DRYCC_NSQD_ADDRS
             value: "{{range $i := until $nsqdNodeCount}}drycc-nsqd-{{$i}}.drycc-nsqd.{{ $.Release.Namespace }}.svc.cluster.local:{{$.Values.nsqd.tcp_port}}{{if lt (add 1 $i) $nsqdNodeCount}},{{end}}{{end}}"
+          {{- range $key, $value := .Values.environment }}
+          - name: {{ $key }}
+            value: {{ $value | quote }}
+          {{- end }}
           volumeMounts:
             - mountPath: /etc/slugrunner
               name: slugrunner-config

charts/controller/values.yaml

@@ -19,6 +19,12 @@ k8s_api_verify_tls: "true"
 # Set storageClassName, It is used for application mount.
 app_storage_class: ""
 
+# Any custom controller environment variables
+# can be specified as key-value pairs under environment
+# this is usually a non required setting.
+environment:
+  RESERVED_NAMES: "drycc, drycc-builder, drycc-monitor-grafana"
+
 nsqd:
   replicas: 1
   tcp_port: 4150

rootfs/api/models/app.py

@@ -26,7 +26,7 @@
 from api.models.tls import TLS
 from api.models.appsettings import AppSettings
 from api.models.volume import Volume
-from api.utils import generate_app_name, async_run
+from api.utils import generate_app_name, apply_tasks
 from scheduler import KubeHTTPException, KubeException
 
 logger = logging.getLogger(__name__)
@@ -383,7 +383,7 @@ def restart(self, **kwargs):  # noqa
                 ) for pod in self.list_pods(**kwargs)
             ]
 
-            async_run(tasks)
+            apply_tasks(tasks)
         except Exception as e:
             err = "warning, some pods failed to stop:\n{}".format(str(e))
             self.log(err, logging.WARNING)
@@ -528,7 +528,7 @@ def _scale_pods(self, scale_types):
             # create the application config in k8s (secret in this case) for all deploy objects
             self.set_application_config(release)
 
-            async_run(tasks)
+            apply_tasks(tasks)
         except Exception as e:
             err = '(scale): {}'.format(e)
             self.log(err, logging.ERROR)
@@ -619,7 +619,7 @@ def deploy(self, release, force_deploy=False, rollback_on_failure=True):  # noqa
             ]
 
             try:
-                async_run(tasks)
+                apply_tasks(tasks)
             except KubeException as e:
                 # Don't rollback if the previous release doesn't have a build which means
                 # this is the first build and all the previous releases are just config changes.

rootfs/api/settings/production.py

@@ -412,6 +412,9 @@
 LDAP_USER_FILTER = os.environ.get('LDAP_USER_FILTER', 'username')
 LDAP_GROUP_BASEDN = os.environ.get('LDAP_GROUP_BASEDN', '')
 LDAP_GROUP_FILTER = os.environ.get('LDAP_GROUP_FILTER', '')
+LDAP_ACTIVE_GROUP = os.environ.get('LDAP_ACTIVE_GROUP', '')
+LDAP_STAFF_GROUP = os.environ.get('LDAP_STAFF_GROUP', '')
+LDAP_SUPERUSER_GROUP = os.environ.get('LDAP_SUPERUSER_GROUP', '')
 
 # Django LDAP backend configuration.
 # See https://pythonhosted.org/django-auth-ldap/reference.html
@@ -435,6 +438,11 @@
         scope=ldap.SCOPE_SUBTREE,
         filterstr="(%s)" % LDAP_GROUP_FILTER
     )
+    AUTH_LDAP_USER_FLAGS_BY_GROUP = {
+        'is_active': LDAP_ACTIVE_GROUP,
+        'is_staff': LDAP_STAFF_GROUP,
+        'is_superuser': LDAP_SUPERUSER_GROUP,
+    }
     AUTH_LDAP_GROUP_TYPE = GroupOfNamesType()
     AUTH_LDAP_USER_ATTR_MAP = {
         "first_name": "givenName",

rootfs/api/utils.py

@@ -1,7 +1,6 @@
 """
 Helper functions used by the Drycc server.
 """
-import asyncio
 import base64
 import concurrent
 import hashlib
@@ -145,43 +144,22 @@ def dict_merge(origin, merge):
     return result
 
 
-def async_run(tasks):
+def apply_tasks(tasks):
     """
     run a group of tasks async
     Requires the tasks arg to be a list of functools.partial()
     """
     if not tasks:
         return
 
-    # start a new async event loop
-    loop = asyncio.get_event_loop()
-    # https://github.com/python/asyncio/issues/258
     executor = concurrent.futures.ThreadPoolExecutor(5)
-    loop.set_default_executor(executor)
-
-    async_tasks = [asyncio.ensure_future(async_task(task, loop)) for task in tasks]
-    # run tasks in parallel
-    loop.run_until_complete(asyncio.wait(async_tasks))
-    # deal with errors (exceptions, etc)
-    for task in async_tasks:
-        error = task.exception()
+    for future in [executor.submit(task) for task in tasks]:
+        error = future.exception()
         if error is not None:
             raise error
-
     executor.shutdown(wait=True)
 
 
-async def async_task(params, loop):
-    """
-    Perform a task asynchronously.
-    """
-    # get the calling function
-    logger.debug('Running {}'.format(params))
-    # This executes a task in its own thread (in parallel)
-    await loop.run_in_executor(None, params)
-    logger.debug('Finished running {}'.format(params))
-
-
 if __name__ == "__main__":
     import doctest
     doctest.testmod()

rootfs/bin/boot

@@ -30,7 +30,7 @@ python /app/manage.py migrate --noinput
 # spawn a gunicorn server in the background
 echo ""
 echo "Starting up Gunicorn"
-gunicorn -c /app/drycc/gunicorn/config.py api.wsgi &
+gunicorn -c /app/drycc/gunicorn/config.py drycc.gunicorn.wsgi &
 
 echo ""
 echo "Loading database information to Kubernetes in the background"

rootfs/drycc/gunicorn/wsgi.py

@@ -0,0 +1,7 @@
+from tornado.wsgi import WSGIContainer
+
+
+from api.wsgi import application as handler
+
+
+application = WSGIContainer(handler)

rootfs/drycc/wsgi.py

@@ -0,0 +1,7 @@
+from tornado.wsgi import WSGIContainer
+
+
+from api.wsgi import application as handler
+
+
+application = WSGIContainer(handler)

rootfs/requirements.txt

@@ -15,7 +15,6 @@ morph==0.1.4
 ndg-httpsclient==0.5.1
 packaging==20.4
 pyasn1==0.4.8
-tornado==5.1.1
 pynsq==0.9.0
 psycopg2-binary==2.8.5
 pyldap==3.0.0.post1