Skip to content

Commit 447f1dc

Browse files
author
Matthew Fisher
committed
Merge pull request #2993 from bacongobbler/2766-query-fleet-state
ref(controller): query fleet state
2 parents aa58135 + 3c462b7 commit 447f1dc

14 files changed

Lines changed: 389 additions & 156 deletions

File tree

‎controller/api/models.py‎

Lines changed: 45 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@
2424
from django.db.models.signals import post_delete, post_save
2525
from django.dispatch import receiver
2626
from django.utils.encoding import python_2_unicode_compatible
27-
from django_fsm import FSMField, transition
28-
from django_fsm.signals import post_transition
2927
from docker.utils import utils as dockerutils
3028
from json_field.fields import JSONField
3129
import requests
@@ -39,6 +37,25 @@
3937
logger = logging.getLogger(__name__)
4038

4139

40+
def close_db_connections(func, *args, **kwargs):
41+
"""
42+
Decorator to explicitly close db connections during threaded execution
43+
44+
Note this is necessary to work around:
45+
https://code.djangoproject.com/ticket/22420
46+
"""
47+
def _close_db_connections(*args, **kwargs):
48+
ret = None
49+
try:
50+
ret = func(*args, **kwargs)
51+
finally:
52+
from django.db import connections
53+
for conn in connections.all():
54+
conn.close()
55+
return ret
56+
return _close_db_connections
57+
58+
4259
def log_event(app, msg, level=logging.INFO):
4360
msg = "{}: {}".format(app.id, msg)
4461
logger.log(level, msg) # django logger
@@ -129,19 +146,16 @@ class App(UuidAuditedModel):
129146
class Meta:
130147
permissions = (('use_app', 'Can use app'),)
131148

132-
def __str__(self):
133-
return self.id
134-
135-
def _get_scheduler(self, *args, **kwargs):
136-
module_name = 'scheduler.' + settings.SCHEDULER_MODULE
137-
mod = importlib.import_module(module_name)
138-
149+
@property
150+
def _scheduler(self):
151+
mod = importlib.import_module(settings.SCHEDULER_MODULE)
139152
return mod.SchedulerClient(settings.SCHEDULER_TARGET,
140153
settings.SCHEDULER_AUTH,
141154
settings.SCHEDULER_OPTIONS,
142155
settings.SSH_PRIVATE_KEY)
143156

144-
_scheduler = property(_get_scheduler)
157+
def __str__(self):
158+
return self.id
145159

146160
@property
147161
def url(self):
@@ -167,8 +181,11 @@ def create(self, *args, **kwargs):
167181

168182
def delete(self, *args, **kwargs):
169183
"""Delete this application including all containers"""
170-
for c in self.container_set.exclude(type='run'):
171-
c.destroy()
184+
try:
185+
# attempt to remove containers from the scheduler
186+
self._destroy_containers([c for c in self.container_set.exclude(type='run')])
187+
except RuntimeError:
188+
pass
172189
self._clean_app_logs()
173190
return super(App, self).delete(*args, **kwargs)
174191

@@ -246,13 +263,13 @@ def _start_containers(self, to_add):
246263
start_threads.append(threading.Thread(target=c.start))
247264
[t.start() for t in create_threads]
248265
[t.join() for t in create_threads]
249-
if set([c.state for c in to_add]) != set([Container.CREATED]):
266+
if set([c.state for c in to_add]) != set(['created']):
250267
err = 'aborting, failed to create some containers'
251268
log_event(self, err, logging.ERROR)
252269
raise RuntimeError(err)
253270
[t.start() for t in start_threads]
254271
[t.join() for t in start_threads]
255-
if set([c.state for c in to_add]) != set([Container.UP]):
272+
if set([c.state for c in to_add]) != set(['up']):
256273
err = 'warning, some containers failed to start'
257274
log_event(self, err, logging.WARNING)
258275

@@ -266,8 +283,8 @@ def _destroy_containers(self, to_destroy):
266283
destroy_threads.append(threading.Thread(target=c.destroy))
267284
[t.start() for t in destroy_threads]
268285
[t.join() for t in destroy_threads]
269-
[c.delete() for c in to_destroy if c.state == Container.DESTROYED]
270-
if set([c.state for c in to_destroy]) != set([Container.DESTROYED]):
286+
[c.delete() for c in to_destroy if c.state == 'destroyed']
287+
if set([c.state for c in to_destroy]) != set(['destroyed']):
271288
err = 'aborting, failed to destroy some containers'
272289
log_event(self, err, logging.ERROR)
273290
raise RuntimeError(err)
@@ -359,30 +376,20 @@ class Container(UuidAuditedModel):
359376
"""
360377
Docker container used to securely host an application process.
361378
"""
362-
INITIALIZED = 'initialized'
363-
CREATED = 'created'
364-
UP = 'up'
365-
DOWN = 'down'
366-
DESTROYED = 'destroyed'
367-
CRASHED = 'crashed'
368-
ERROR = 'error'
369-
STATE_CHOICES = (
370-
(INITIALIZED, 'initialized'),
371-
(CREATED, 'created'),
372-
(UP, 'up'),
373-
(DOWN, 'down'),
374-
(DESTROYED, 'destroyed'),
375-
(CRASHED, 'crashed'),
376-
(ERROR, 'error'),
377-
)
378379

379380
owner = models.ForeignKey(settings.AUTH_USER_MODEL)
380381
app = models.ForeignKey('App')
381382
release = models.ForeignKey('Release')
382383
type = models.CharField(max_length=128, blank=False)
383384
num = models.PositiveIntegerField()
384-
state = FSMField(default=INITIALIZED, choices=STATE_CHOICES,
385-
protected=True, propagate=False)
385+
386+
@property
387+
def _scheduler(self):
388+
return self.app._scheduler
389+
390+
@property
391+
def state(self):
392+
return self._scheduler.state(self._job_id).name
386393

387394
def short_name(self):
388395
return "{}.{}.{}".format(self.app.id, self.type, self.num)
@@ -405,11 +412,6 @@ def _get_job_id(self):
405412

406413
_job_id = property(_get_job_id)
407414

408-
def _get_scheduler(self):
409-
return self.app._scheduler
410-
411-
_scheduler = property(_get_scheduler)
412-
413415
def _get_command(self):
414416
try:
415417
# if this is not procfile-based app, ensure they cannot break out
@@ -434,7 +436,7 @@ def clone(self, release):
434436
num=self.num)
435437
return c
436438

437-
@transition(field=state, source=INITIALIZED, target=CREATED, on_error=ERROR)
439+
@close_db_connections
438440
def create(self):
439441
image = self.release.image
440442
kwargs = {'memory': self.release.config.memory,
@@ -452,7 +454,7 @@ def create(self):
452454
log_event(self.app, err, logging.ERROR)
453455
raise
454456

455-
@transition(field=state, source=[CREATED, UP, DOWN], target=UP, on_error=CRASHED)
457+
@close_db_connections
456458
def start(self):
457459
job_id = self._job_id
458460
try:
@@ -462,7 +464,7 @@ def start(self):
462464
log_event(self.app, err, logging.WARNING)
463465
raise
464466

465-
@transition(field=state, source=UP, target=DOWN, on_error=ERROR)
467+
@close_db_connections
466468
def stop(self):
467469
job_id = self._job_id
468470
try:
@@ -472,7 +474,7 @@ def stop(self):
472474
log_event(self.app, err, logging.ERROR)
473475
raise
474476

475-
@transition(field=state, source='*', target=DESTROYED, on_error=ERROR)
477+
@close_db_connections
476478
def destroy(self):
477479
job_id = self._job_id
478480
try:
@@ -942,17 +944,6 @@ def create_auth_token(sender, instance=None, created=False, **kwargs):
942944
if created:
943945
Token.objects.create(user=instance)
944946

945-
946-
# save FSM transitions as they happen
947-
def _save_transition(**kwargs):
948-
kwargs['instance'].save()
949-
# close database connections after transition
950-
# to avoid leaking connections inside threads
951-
from django.db import connection
952-
connection.close()
953-
954-
post_transition.connect(_save_transition)
955-
956947
# wire up etcd publishing if we can connect
957948
try:
958949
_etcd_client = etcd.Client(host=settings.ETCD_HOST, port=int(settings.ETCD_PORT))

‎controller/api/serializers.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,7 @@ class ContainerSerializer(ModelSerializer):
205205
class Meta:
206206
"""Metadata options for a :class:`ContainerSerializer`."""
207207
model = models.Container
208+
fields = ['owner', 'app', 'release', 'type', 'num', 'state', 'created', 'updated', 'uuid']
208209

209210
def get_release(self, obj):
210211
return "v{}".format(obj.release.version)

‎controller/api/south_migrations/0020_auto__del_field_container_state.py‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
# -*- coding: utf-8 -*-
2+
from south.utils import datetime_utils as datetime
3+
from south.db import db
4+
from south.v2 import SchemaMigration
5+
from django.db import models
6+
7+
8+
class Migration(SchemaMigration):
9+
10+
def forwards(self, orm):
11+
# Deleting field 'Container.state'
12+
db.delete_column(u'api_container', 'state')
13+
14+
15+
def backwards(self, orm):
16+
# Adding field 'Container.state'
17+
db.add_column(u'api_container', 'state',
18+
self.gf('django_fsm.FSMField')(default=u'initialized', max_length=50),
19+
keep_default=False)
20+
21+
22+
models = {
23+
u'api.app': {
24+
'Meta': {'object_name': 'App'},
25+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
26+
'id': ('django.db.models.fields.SlugField', [], {'default': "'verbal-autoharp'", 'unique': 'True', 'max_length': '64'}),
27+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
28+
'structure': ('json_field.fields.JSONField', [], {'default': '{}', 'blank': 'True'}),
29+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
30+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
31+
},
32+
u'api.build': {
33+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Build'},
34+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
35+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
36+
'dockerfile': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
37+
'image': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
38+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
39+
'procfile': ('json_field.fields.JSONField', [], {'default': '{}', 'blank': 'True'}),
40+
'sha': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'}),
41+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
42+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
43+
},
44+
u'api.config': {
45+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Config'},
46+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
47+
'cpu': ('json_field.fields.JSONField', [], {'default': '{}', 'blank': 'True'}),
48+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
49+
'memory': ('json_field.fields.JSONField', [], {'default': '{}', 'blank': 'True'}),
50+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
51+
'tags': ('json_field.fields.JSONField', [], {'default': '{}', 'blank': 'True'}),
52+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
53+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'}),
54+
'values': ('json_field.fields.JSONField', [], {'default': '{}', 'blank': 'True'})
55+
},
56+
u'api.container': {
57+
'Meta': {'ordering': "[u'created']", 'object_name': 'Container'},
58+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
59+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
60+
'num': ('django.db.models.fields.PositiveIntegerField', [], {}),
61+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
62+
'release': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Release']"}),
63+
'type': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
64+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
65+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
66+
},
67+
u'api.domain': {
68+
'Meta': {'object_name': 'Domain'},
69+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
70+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
71+
'domain': ('django.db.models.fields.TextField', [], {'unique': 'True'}),
72+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
73+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
74+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
75+
},
76+
u'api.key': {
77+
'Meta': {'unique_together': "((u'owner', u'id'),)", 'object_name': 'Key'},
78+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
79+
'id': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
80+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
81+
'public': ('django.db.models.fields.TextField', [], {'unique': 'True'}),
82+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
83+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
84+
},
85+
u'api.push': {
86+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Push'},
87+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
88+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
89+
'fingerprint': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
90+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
91+
'receive_repo': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
92+
'receive_user': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
93+
'sha': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
94+
'ssh_connection': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
95+
'ssh_original_command': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
96+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
97+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
98+
},
99+
u'api.release': {
100+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'version'),)", 'object_name': 'Release'},
101+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
102+
'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Build']", 'null': 'True'}),
103+
'config': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Config']"}),
104+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
105+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
106+
'summary': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
107+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
108+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'}),
109+
'version': ('django.db.models.fields.PositiveIntegerField', [], {})
110+
},
111+
u'auth.group': {
112+
'Meta': {'object_name': 'Group'},
113+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
114+
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
115+
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
116+
},
117+
u'auth.permission': {
118+
'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
119+
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
120+
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
121+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
122+
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
123+
},
124+
u'auth.user': {
125+
'Meta': {'object_name': 'User'},
126+
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
127+
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
128+
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
129+
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
130+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
131+
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
132+
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
133+
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
134+
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
135+
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
136+
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
137+
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
138+
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
139+
},
140+
u'contenttypes.contenttype': {
141+
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
142+
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
143+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
144+
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
145+
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
146+
}
147+
}
148+
149+
complete_apps = ['api']

‎controller/api/tests/test_container.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212

1313
from django.contrib.auth.models import User
1414
from django.test import TransactionTestCase
15-
from django_fsm import TransitionNotAllowed
15+
from scheduler.states import TransitionNotAllowed
1616
from rest_framework.authtoken.models import Token
1717

1818
from api.models import App, Build, Container, Release

‎controller/api/tests/test_scheduler.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,13 @@ def setUp(self):
3030
chaos.START_ERROR_RATE = 0
3131
chaos.STOP_ERROR_RATE = 0
3232
# use chaos scheduler
33-
settings.SCHEDULER_MODULE = 'chaos'
33+
settings.SCHEDULER_MODULE = 'scheduler.chaos'
3434
# provide mock authentication used for run commands
3535
settings.SSH_PRIVATE_KEY = '<some-ssh-private-key>'
3636

3737
def tearDown(self):
3838
# reset for subsequent tests
39-
settings.SCHEDULER_MODULE = 'mock'
39+
settings.SCHEDULER_MODULE = 'scheduler.mock'
4040
settings.SSH_PRIVATE_KEY = ''
4141

4242
def test_create_chaos(self):

0 commit comments

Comments
 (0)