Merge pull request #98 from opdemand/log-aggregation

add logs endpoint, views and tests along with `deis logs` cli command re #20

Author: mboersma

.gitignore

@@ -40,6 +40,9 @@ nosetests.xml
 # Deis' config file
 deis/local_settings.py
 
+# deis application logs
+logs/
+
 # Misc.
 .DS_Store
 htmlcov/

api/models.py

@@ -10,6 +10,7 @@
 import importlib
 import json
 import os
+import subprocess
 import yaml
 
 from celery.canvas import group
@@ -436,6 +437,14 @@ def converge(self, databag):
         job.apply_async().join()
         return databag
 
+    def logs(self):
+        """Return aggregated log data for this formation."""
+        path = os.path.join(settings.DEIS_LOG_DIR, self.id + '.log')
+        if not os.path.exists(path):
+            raise EnvironmentError('Could not locate logs')
+        data = subprocess.check_output(['tail', '-n', str(settings.LOG_LINES), path])
+        return data
+
     def destroy(self):
         """Create subtasks to terminate all nodes in parallel."""
         all_layers = self.layer_set.all()

api/tests/formation.py

@@ -7,8 +7,10 @@
 from __future__ import unicode_literals
 
 import json
+import os.path
 
 from django.test import TestCase
+from deis import settings
 
 
 class FormationTest(TestCase):
@@ -130,3 +132,21 @@ def test_formation_actions(self):
         self.assertIn('containers', response.data)
         self.assertIn('proxy', response.data)
         self.assertIn('release', response.data)
+        # test logs
+        if not os.path.exists(settings.DEIS_LOG_DIR):
+            os.mkdir(settings.DEIS_LOG_DIR)
+        path = os.path.join(settings.DEIS_LOG_DIR, formation_id + '.log')
+        with open(path, 'w') as f:
+            f.write(FAKE_LOG_DATA)
+        url = '/api/formations/{formation_id}/logs'.format(**locals())
+        response = self.client.post(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data, FAKE_LOG_DATA)
+
+
+FAKE_LOG_DATA = """
+2013-08-15 12:41:25 [33454] [INFO] Starting gunicorn 17.5
+2013-08-15 12:41:25 [33454] [INFO] Listening at: http://0.0.0.0:5000 (33454)
+2013-08-15 12:41:25 [33454] [INFO] Using worker: sync
+2013-08-15 12:41:25 [33457] [INFO] Booting worker with pid 33457
+"""

api/urls.py

@@ -283,6 +283,8 @@
         views.FormationViewSet.as_view({'post': 'calculate'})),
     url(r'^formations/(?P<id>[a-z0-9-]+)/converge/?',
         views.FormationViewSet.as_view({'post': 'converge'})),
+    url(r'^formations/(?P<id>[a-z0-9-]+)/logs/?',
+        views.FormationViewSet.as_view({'post': 'logs'})),
     # formation base endpoint
     url(r'^formations/(?P<id>[a-z0-9-]+)/?',
         views.FormationViewSet.as_view({'get': 'retrieve', 'delete': 'destroy'})),

api/views.py

@@ -223,6 +223,12 @@ def converge(self, request, **kwargs):
         return Response(databag, status=status.HTTP_200_OK,
                         content_type='application/json')
 
+    def logs(self, request, **kwargs):
+        formation = self.get_object()
+        logs = formation.logs()
+        return Response(logs, status=status.HTTP_200_OK,
+                        content_type='text/plain')
+
     def destroy(self, request, **kwargs):
         formation = self.get_object()
         formation.destroy()

client/deis.py

@@ -1026,6 +1026,22 @@ def layers_scale(self, args):
         else:
             print('Error!', response.text)
 
+    def logs(self, args):
+        """
+        Retrieve the most recent log events
+
+        Usage: deis logs
+        """
+        formation = args.get('--formation')
+        if not formation:
+            formation = self._session.formation
+        response = self._dispatch('post',
+                                  "/api/formations/{}/logs".format(formation))
+        if response.status_code == requests.codes.ok:  # @UndefinedVariable
+            print(response.json())
+        else:
+            print('Error!', response.text)
+
     def nodes(self, args):
         """
         Valid commands for nodes:

deis/settings.py

@@ -222,6 +222,8 @@
 
 # default deis settings
 CONVERGE_ON_PUSH = True
+DEIS_LOG_DIR = os.path.abspath(os.path.join(__file__, '..', '..', 'logs'))
+LOG_LINES = 1000
 
 # Create a file named "local_settings.py" to contain sensitive settings data
 # such as database configuration, admin email, or passwords and keys. It