Skip to content

Commit a6515c8

Browse files
author
Matthew Fisher
committed
ref(controller): query fleet state
Before, we relied on django-fsm to give us a general idea of what state the container object in the model is at. This did not give users an accurate idea of the state of their containers. Introducing a new .state() method to the scheduler as well as changing Container.state to call it's own scheduler's .state() method allows users to directly understand what state their containers are in.
1 parent 58c5e98 commit a6515c8

13 files changed

Lines changed: 366 additions & 151 deletions

File tree

‎controller/api/models.py‎

Lines changed: 40 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@
2424
from django.db.models.signals import post_delete, post_save
2525
from django.dispatch import receiver
2626
from django.utils.encoding import python_2_unicode_compatible
27-
from django_fsm import FSMField, transition
28-
from django_fsm.signals import post_transition
2927
from docker.utils import utils as dockerutils
3028
from json_field.fields import JSONField
3129
import requests
@@ -39,6 +37,25 @@
3937
logger = logging.getLogger(__name__)
4038

4139

40+
def close_db_connections(func, *args, **kwargs):
41+
"""
42+
Decorator to explicitly close db connections during threaded execution
43+
44+
Note this is necessary to work around:
45+
https://code.djangoproject.com/ticket/22420
46+
"""
47+
def _close_db_connections(*args, **kwargs):
48+
ret = None
49+
try:
50+
ret = func(*args, **kwargs)
51+
finally:
52+
from django.db import connections
53+
for conn in connections.all():
54+
conn.close()
55+
return ret
56+
return _close_db_connections
57+
58+
4259
def log_event(app, msg, level=logging.INFO):
4360
msg = "{}: {}".format(app.id, msg)
4461
logger.log(level, msg) # django logger
@@ -129,19 +146,16 @@ class App(UuidAuditedModel):
129146
class Meta:
130147
permissions = (('use_app', 'Can use app'),)
131148

132-
def __str__(self):
133-
return self.id
134-
135-
def _get_scheduler(self, *args, **kwargs):
136-
module_name = 'scheduler.' + settings.SCHEDULER_MODULE
137-
mod = importlib.import_module(module_name)
138-
149+
@property
150+
def _scheduler(self):
151+
mod = importlib.import_module(settings.SCHEDULER_MODULE)
139152
return mod.SchedulerClient(settings.SCHEDULER_TARGET,
140153
settings.SCHEDULER_AUTH,
141154
settings.SCHEDULER_OPTIONS,
142155
settings.SSH_PRIVATE_KEY)
143156

144-
_scheduler = property(_get_scheduler)
157+
def __str__(self):
158+
return self.id
145159

146160
@property
147161
def url(self):
@@ -246,13 +260,13 @@ def _start_containers(self, to_add):
246260
start_threads.append(threading.Thread(target=c.start))
247261
[t.start() for t in create_threads]
248262
[t.join() for t in create_threads]
249-
if set([c.state for c in to_add]) != set([Container.CREATED]):
263+
if set([c.state for c in to_add]) != set(['created']):
250264
err = 'aborting, failed to create some containers'
251265
log_event(self, err, logging.ERROR)
252266
raise RuntimeError(err)
253267
[t.start() for t in start_threads]
254268
[t.join() for t in start_threads]
255-
if set([c.state for c in to_add]) != set([Container.UP]):
269+
if set([c.state for c in to_add]) != set(['up']):
256270
err = 'warning, some containers failed to start'
257271
log_event(self, err, logging.WARNING)
258272

@@ -266,8 +280,8 @@ def _destroy_containers(self, to_destroy):
266280
destroy_threads.append(threading.Thread(target=c.destroy))
267281
[t.start() for t in destroy_threads]
268282
[t.join() for t in destroy_threads]
269-
[c.delete() for c in to_destroy if c.state == Container.DESTROYED]
270-
if set([c.state for c in to_destroy]) != set([Container.DESTROYED]):
283+
[c.delete() for c in to_destroy if c.state == 'destroyed']
284+
if set([c.state for c in to_destroy]) != set(['destroyed']):
271285
err = 'aborting, failed to destroy some containers'
272286
log_event(self, err, logging.ERROR)
273287
raise RuntimeError(err)
@@ -359,30 +373,20 @@ class Container(UuidAuditedModel):
359373
"""
360374
Docker container used to securely host an application process.
361375
"""
362-
INITIALIZED = 'initialized'
363-
CREATED = 'created'
364-
UP = 'up'
365-
DOWN = 'down'
366-
DESTROYED = 'destroyed'
367-
CRASHED = 'crashed'
368-
ERROR = 'error'
369-
STATE_CHOICES = (
370-
(INITIALIZED, 'initialized'),
371-
(CREATED, 'created'),
372-
(UP, 'up'),
373-
(DOWN, 'down'),
374-
(DESTROYED, 'destroyed'),
375-
(CRASHED, 'crashed'),
376-
(ERROR, 'error'),
377-
)
378376

379377
owner = models.ForeignKey(settings.AUTH_USER_MODEL)
380378
app = models.ForeignKey('App')
381379
release = models.ForeignKey('Release')
382380
type = models.CharField(max_length=128, blank=False)
383381
num = models.PositiveIntegerField()
384-
state = FSMField(default=INITIALIZED, choices=STATE_CHOICES,
385-
protected=True, propagate=False)
382+
383+
@property
384+
def _scheduler(self):
385+
return self.app._scheduler
386+
387+
@property
388+
def state(self):
389+
return self._scheduler.state(self._job_id).name
386390

387391
def short_name(self):
388392
return "{}.{}.{}".format(self.app.id, self.type, self.num)
@@ -405,11 +409,6 @@ def _get_job_id(self):
405409

406410
_job_id = property(_get_job_id)
407411

408-
def _get_scheduler(self):
409-
return self.app._scheduler
410-
411-
_scheduler = property(_get_scheduler)
412-
413412
def _get_command(self):
414413
try:
415414
# if this is not procfile-based app, ensure they cannot break out
@@ -434,7 +433,7 @@ def clone(self, release):
434433
num=self.num)
435434
return c
436435

437-
@transition(field=state, source=INITIALIZED, target=CREATED, on_error=ERROR)
436+
@close_db_connections
438437
def create(self):
439438
image = self.release.image
440439
kwargs = {'memory': self.release.config.memory,
@@ -452,7 +451,7 @@ def create(self):
452451
log_event(self.app, err, logging.ERROR)
453452
raise
454453

455-
@transition(field=state, source=[CREATED, UP, DOWN], target=UP, on_error=CRASHED)
454+
@close_db_connections
456455
def start(self):
457456
job_id = self._job_id
458457
try:
@@ -462,7 +461,7 @@ def start(self):
462461
log_event(self.app, err, logging.WARNING)
463462
raise
464463

465-
@transition(field=state, source=UP, target=DOWN, on_error=ERROR)
464+
@close_db_connections
466465
def stop(self):
467466
job_id = self._job_id
468467
try:
@@ -472,7 +471,7 @@ def stop(self):
472471
log_event(self.app, err, logging.ERROR)
473472
raise
474473

475-
@transition(field=state, source='*', target=DESTROYED, on_error=ERROR)
474+
@close_db_connections
476475
def destroy(self):
477476
job_id = self._job_id
478477
try:
@@ -942,17 +941,6 @@ def create_auth_token(sender, instance=None, created=False, **kwargs):
942941
if created:
943942
Token.objects.create(user=instance)
944943

945-
946-
# save FSM transitions as they happen
947-
def _save_transition(**kwargs):
948-
kwargs['instance'].save()
949-
# close database connections after transition
950-
# to avoid leaking connections inside threads
951-
from django.db import connection
952-
connection.close()
953-
954-
post_transition.connect(_save_transition)
955-
956944
# wire up etcd publishing if we can connect
957945
try:
958946
_etcd_client = etcd.Client(host=settings.ETCD_HOST, port=int(settings.ETCD_PORT))

‎controller/api/serializers.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,7 @@ class ContainerSerializer(ModelSerializer):
205205
class Meta:
206206
"""Metadata options for a :class:`ContainerSerializer`."""
207207
model = models.Container
208+
fields = ['owner', 'app', 'release', 'type', 'num', 'state', 'created', 'updated', 'uuid']
208209

209210
def get_release(self, obj):
210211
return "v{}".format(obj.release.version)

‎controller/api/south_migrations/0020_auto__del_field_container_state.py‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
# -*- coding: utf-8 -*-
2+
from south.utils import datetime_utils as datetime
3+
from south.db import db
4+
from south.v2 import SchemaMigration
5+
from django.db import models
6+
7+
8+
class Migration(SchemaMigration):
9+
10+
def forwards(self, orm):
11+
# Deleting field 'Container.state'
12+
db.delete_column(u'api_container', 'state')
13+
14+
15+
def backwards(self, orm):
16+
# Adding field 'Container.state'
17+
db.add_column(u'api_container', 'state',
18+
self.gf('django_fsm.FSMField')(default=u'initialized', max_length=50),
19+
keep_default=False)
20+
21+
22+
models = {
23+
u'api.app': {
24+
'Meta': {'object_name': 'App'},
25+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
26+
'id': ('django.db.models.fields.SlugField', [], {'default': "'verbal-autoharp'", 'unique': 'True', 'max_length': '64'}),
27+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
28+
'structure': ('json_field.fields.JSONField', [], {'default': '{}', 'blank': 'True'}),
29+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
30+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
31+
},
32+
u'api.build': {
33+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Build'},
34+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
35+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
36+
'dockerfile': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
37+
'image': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
38+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
39+
'procfile': ('json_field.fields.JSONField', [], {'default': '{}', 'blank': 'True'}),
40+
'sha': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'}),
41+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
42+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
43+
},
44+
u'api.config': {
45+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Config'},
46+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
47+
'cpu': ('json_field.fields.JSONField', [], {'default': '{}', 'blank': 'True'}),
48+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
49+
'memory': ('json_field.fields.JSONField', [], {'default': '{}', 'blank': 'True'}),
50+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
51+
'tags': ('json_field.fields.JSONField', [], {'default': '{}', 'blank': 'True'}),
52+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
53+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'}),
54+
'values': ('json_field.fields.JSONField', [], {'default': '{}', 'blank': 'True'})
55+
},
56+
u'api.container': {
57+
'Meta': {'ordering': "[u'created']", 'object_name': 'Container'},
58+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
59+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
60+
'num': ('django.db.models.fields.PositiveIntegerField', [], {}),
61+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
62+
'release': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Release']"}),
63+
'type': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
64+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
65+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
66+
},
67+
u'api.domain': {
68+
'Meta': {'object_name': 'Domain'},
69+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
70+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
71+
'domain': ('django.db.models.fields.TextField', [], {'unique': 'True'}),
72+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
73+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
74+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
75+
},
76+
u'api.key': {
77+
'Meta': {'unique_together': "((u'owner', u'id'),)", 'object_name': 'Key'},
78+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
79+
'id': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
80+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
81+
'public': ('django.db.models.fields.TextField', [], {'unique': 'True'}),
82+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
83+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
84+
},
85+
u'api.push': {
86+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Push'},
87+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
88+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
89+
'fingerprint': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
90+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
91+
'receive_repo': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
92+
'receive_user': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
93+
'sha': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
94+
'ssh_connection': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
95+
'ssh_original_command': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
96+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
97+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
98+
},
99+
u'api.release': {
100+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'version'),)", 'object_name': 'Release'},
101+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
102+
'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Build']", 'null': 'True'}),
103+
'config': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Config']"}),
104+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
105+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
106+
'summary': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
107+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
108+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'}),
109+
'version': ('django.db.models.fields.PositiveIntegerField', [], {})
110+
},
111+
u'auth.group': {
112+
'Meta': {'object_name': 'Group'},
113+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
114+
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
115+
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
116+
},
117+
u'auth.permission': {
118+
'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
119+
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
120+
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
121+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
122+
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
123+
},
124+
u'auth.user': {
125+
'Meta': {'object_name': 'User'},
126+
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
127+
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
128+
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
129+
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
130+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
131+
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
132+
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
133+
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
134+
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
135+
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
136+
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
137+
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
138+
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
139+
},
140+
u'contenttypes.contenttype': {
141+
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
142+
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
143+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
144+
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
145+
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
146+
}
147+
}
148+
149+
complete_apps = ['api']

‎controller/api/tests/test_container.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212

1313
from django.contrib.auth.models import User
1414
from django.test import TransactionTestCase
15-
from django_fsm import TransitionNotAllowed
15+
from scheduler.states import TransitionNotAllowed
1616
from rest_framework.authtoken.models import Token
1717

1818
from api.models import App, Build, Container, Release

‎controller/api/tests/test_scheduler.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,13 @@ def setUp(self):
3030
chaos.START_ERROR_RATE = 0
3131
chaos.STOP_ERROR_RATE = 0
3232
# use chaos scheduler
33-
settings.SCHEDULER_MODULE = 'chaos'
33+
settings.SCHEDULER_MODULE = 'scheduler.chaos'
3434
# provide mock authentication used for run commands
3535
settings.SSH_PRIVATE_KEY = '<some-ssh-private-key>'
3636

3737
def tearDown(self):
3838
# reset for subsequent tests
39-
settings.SCHEDULER_MODULE = 'mock'
39+
settings.SCHEDULER_MODULE = 'scheduler.mock'
4040
settings.SSH_PRIVATE_KEY = ''
4141

4242
def test_create_chaos(self):

‎controller/deis/settings.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,6 @@
136136
'django.contrib.sites',
137137
'django.contrib.staticfiles',
138138
# Third-party apps
139-
'django_fsm',
140139
'guardian',
141140
'json_field',
142141
'gunicorn',
@@ -276,7 +275,7 @@
276275
DEIS_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S%Z'
277276

278277
# default scheduler settings
279-
SCHEDULER_MODULE = 'mock'
278+
SCHEDULER_MODULE = 'scheduler.mock'
280279
SCHEDULER_TARGET = '' # path to scheduler endpoint (e.g. /var/run/fleet.sock)
281280
SCHEDULER_AUTH = ''
282281
SCHEDULER_OPTIONS = {}

0 commit comments

Comments
 (0)