feat(controller): multi pod restart uses async

Author: duanhongyi

rootfs/api/models/app.py

@@ -697,7 +697,6 @@ def verify_application_health(self, **kwargs):
         only run after kubernetes has reported all pods as healthy
         """
         # Bail out early if the application is not routable
-        release = self.release_set.filter(failed=False).latest()
         app_settings = self.appsettings_set.latest()
         if not kwargs.get('routable', False) and app_settings.routable:
             return

rootfs/api/settings/celery.py

@@ -26,6 +26,8 @@ class Config:
 app.config_from_object(Config)
 app.conf.update(
     task_routes={
+        'api.tasks.scale_app': {'queue': 'priority.high'},
+        'api.tasks.restart_app': {'queue': 'priority.high'},
         'api.tasks.retrieve_resource': {'queue': 'priority.high'},
     },
 )

rootfs/api/tasks.py

@@ -61,3 +61,17 @@ def scale_app(app, user, structure):
         app.scale(user, structure)
     finally:
         signals.request_finished.send(sender=task_id)
+
+
+@shared_task(
+    autoretry_for=(ServiceUnavailable, ),
+    retry_jitter=True,
+    retry_kwargs={'max_retries': 3}
+)
+def restart_app(app, **kwargs):
+    task_id = uuid.uuid4().hex
+    signals.request_started.send(sender=task_id)
+    try:
+        app.restart(**kwargs)
+    finally:
+        signals.request_finished.send(sender=task_id)

rootfs/api/views.py

@@ -20,7 +20,7 @@
 from rest_framework.viewsets import GenericViewSet
 
 from api import influxdb, models, permissions, serializers, viewsets
-from api.tasks import scale_app
+from api.tasks import scale_app, restart_app
 from api.models import AlreadyExists, ServiceUnavailable, DryccException, \
     UnprocessableEntity
 
@@ -280,7 +280,11 @@ def list(self, *args, **kwargs):
         return Response(pagination, status=status.HTTP_200_OK)
 
     def restart(self, *args, **kwargs):
-        pods = self.get_app().restart(**kwargs)
+        if "name" in kwargs:  # a single pod uses sync
+            pods = self.get_app().restart(**kwargs)
+        else:  # multi pod uses async
+            restart_app.delay(self.get_app(), **kwargs)
+            pods = self.get_app().list_pods(**kwargs)
         data = self.get_serializer(pods, many=True).data
         # fake out pagination for now
         # pagination = {'results': data, 'count': len(data)}