Skip to content

Commit cbf5e63

Browse files
author
Matthew Fisher
committed
feat(controller): add container state via FSM
This makes a container's state directly managed via a Finite State Machine (see http://en.wikipedia.org/wiki/Finite-state_machine). With this new behaviour, containers can be in one of 5 different states: 1) up 2) down 3) initialized 4) created 5) destroyed A diagram that shows the container's transitions as it goes from state to state would be nice, but is not necessary for this commit. I've noted this as a TODO feature in the documentation. I have also set the container's FSM to protected, so that nobody can directly change the FSM's state. Only a call to the model's functions that have a @transition decorator will be able to change the state. fixes #699
1 parent 43bc5fa commit cbf5e63

7 files changed

Lines changed: 214 additions & 54 deletions

File tree

‎Dockerfile‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,27 @@ RUN apt-get update && \
99
RUN wget -qO- https://raw.github.com/pypa/pip/1.5.4/contrib/get-pip.py | python -
1010

1111
# install requirements before ADD to cache layer and speed build
12-
RUN pip install boto==2.23.0 celery==3.1.8 Django==1.6.2 django-allauth==0.15.0 django-guardian==1.1.1 django-json-field==0.5.5 django-yamlfield==0.5 djangorestframework==2.3.12 dop==0.1.4 gevent==1.0 gunicorn==18.0 paramiko==1.12.1 psycopg2==2.5.2 pycrypto==2.6.1 python-etcd==0.3.0 pyrax==1.6.2 PyYAML==3.10 redis==2.8.0 static==1.0.2 South==0.8.4
12+
RUN pip install boto==2.23.0 \
13+
celery==3.1.8 \
14+
Django==1.6.2 \
15+
django-allauth==0.15.0 \
16+
git+https://github.com/bacongobbler/django-fsm@add-exception-handling \
17+
django-guardian==1.1.1 \
18+
django-json-field==0.5.5 \
19+
django-yamlfield==0.5 \
20+
djangorestframework==2.3.12 \
21+
dop==0.1.6 \
22+
gevent==1.0 \
23+
gunicorn==18.0 \
24+
paramiko==1.12.1 \
25+
psycopg2==2.5.2 \
26+
pycrypto==2.6.1 \
27+
python-etcd==0.3.0 \
28+
pyrax==1.6.2 \
29+
PyYAML==3.10 \
30+
redis==2.8.0 \
31+
static==1.0.2 \
32+
South==0.8.4
1333

1434
# install openssh-client for temporary fleetctl wrapper
1535
RUN apt-get install -yq openssh-client

‎api/migrations/0009_auto__chg_field_container_state.py‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
# -*- coding: utf-8 -*-
2+
from south.utils import datetime_utils as datetime
3+
from south.db import db
4+
from south.v2 import SchemaMigration
5+
from django.db import models
6+
7+
8+
class Migration(SchemaMigration):
9+
10+
def forwards(self, orm):
11+
12+
# Changing field 'Container.state'
13+
db.alter_column(u'api_container', 'state', self.gf('django_fsm.FSMField')(max_length=50))
14+
15+
def backwards(self, orm):
16+
17+
# Changing field 'Container.state'
18+
db.alter_column(u'api_container', 'state', self.gf('django.db.models.fields.CharField')(max_length=64))
19+
20+
models = {
21+
u'api.app': {
22+
'Meta': {'object_name': 'App'},
23+
'cluster': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Cluster']"}),
24+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
25+
'id': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '64'}),
26+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
27+
'structure': ('json_field.fields.JSONField', [], {'default': "u'{}'", 'blank': 'True'}),
28+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
29+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
30+
},
31+
u'api.build': {
32+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Build'},
33+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
34+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
35+
'image': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
36+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
37+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
38+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
39+
},
40+
u'api.cluster': {
41+
'Meta': {'object_name': 'Cluster'},
42+
'auth': ('django.db.models.fields.TextField', [], {}),
43+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
44+
'domain': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
45+
'hosts': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
46+
'id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
47+
'options': ('json_field.fields.JSONField', [], {'default': "u'{}'", 'blank': 'True'}),
48+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
49+
'type': ('django.db.models.fields.CharField', [], {'max_length': '16'}),
50+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
51+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
52+
},
53+
u'api.config': {
54+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Config'},
55+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
56+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
57+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
58+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
59+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'}),
60+
'values': ('json_field.fields.JSONField', [], {'default': "u'{}'", 'blank': 'True'})
61+
},
62+
u'api.container': {
63+
'Meta': {'ordering': "[u'created']", 'object_name': 'Container'},
64+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
65+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
66+
'num': ('django.db.models.fields.PositiveIntegerField', [], {}),
67+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
68+
'release': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Release']"}),
69+
'state': ('django_fsm.FSMField', [], {'default': "u'initialized'", 'max_length': '50'}),
70+
'type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
71+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
72+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
73+
},
74+
u'api.key': {
75+
'Meta': {'unique_together': "((u'owner', u'id'),)", 'object_name': 'Key'},
76+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
77+
'id': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
78+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
79+
'public': ('django.db.models.fields.TextField', [], {'unique': 'True'}),
80+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
81+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
82+
},
83+
u'api.push': {
84+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Push'},
85+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
86+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
87+
'fingerprint': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
88+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
89+
'receive_repo': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
90+
'receive_user': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
91+
'sha': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
92+
'ssh_connection': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
93+
'ssh_original_command': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
94+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
95+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
96+
},
97+
u'api.release': {
98+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'version'),)", 'object_name': 'Release'},
99+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
100+
'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Build']"}),
101+
'config': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Config']"}),
102+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
103+
'image': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
104+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
105+
'summary': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
106+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
107+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'}),
108+
'version': ('django.db.models.fields.PositiveIntegerField', [], {})
109+
},
110+
u'auth.group': {
111+
'Meta': {'object_name': 'Group'},
112+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
113+
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
114+
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
115+
},
116+
u'auth.permission': {
117+
'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
118+
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
119+
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
120+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
121+
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
122+
},
123+
u'auth.user': {
124+
'Meta': {'object_name': 'User'},
125+
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
126+
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
127+
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
128+
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
129+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
130+
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
131+
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
132+
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
133+
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
134+
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
135+
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
136+
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
137+
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
138+
},
139+
u'contenttypes.contenttype': {
140+
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
141+
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
142+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
143+
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
144+
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
145+
}
146+
}
147+
148+
complete_apps = ['api']

‎api/models.py‎

Lines changed: 30 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from django.db.models.signals import post_delete
1919
from django.db.models.signals import post_save
2020
from django.utils.encoding import python_2_unicode_compatible
21+
from django_fsm import FSMField, transition
2122
from json_field.fields import JSONField
2223

2324
from api import fields, tasks
@@ -199,14 +200,25 @@ class Container(UuidAuditedModel):
199200
"""
200201
Docker container used to securely host an application process.
201202
"""
203+
INITIALIZED = 'initialized'
204+
CREATED = 'created'
205+
UP = 'up'
206+
DOWN = 'down'
207+
DESTROYED = 'destroyed'
208+
STATE_CHOICES = (
209+
(INITIALIZED, 'initialized'),
210+
(CREATED, 'created'),
211+
(UP, 'up'),
212+
(DOWN, 'down'),
213+
(DESTROYED, 'destroyed')
214+
)
202215

203216
owner = models.ForeignKey(settings.AUTH_USER_MODEL)
204217
app = models.ForeignKey('App')
205218
release = models.ForeignKey('Release')
206219
type = models.CharField(max_length=128, blank=True)
207220
num = models.PositiveIntegerField()
208-
# TODO: implement fsm
209-
state = models.CharField(max_length=64, default='initializing')
221+
state = FSMField(default=INITIALIZED, choices=STATE_CHOICES, protected=True)
210222

211223
def short_name(self):
212224
if self.type:
@@ -249,24 +261,24 @@ def _get_command(self):
249261

250262
_command = property(_get_command)
251263

264+
@transition(field=state, source=INITIALIZED, target=CREATED)
252265
def create(self):
253266
image = self.release.image
254267
c_type = self.type
255268
self._scheduler.create(self._job_id, image, self._command.format(**locals()))
256-
self.state = 'created'
257-
self.save()
258269

270+
@transition(field=state,
271+
source=[CREATED, UP, DOWN],
272+
target=UP, crashed=DOWN)
259273
def start(self):
260-
self.state = 'starting'
261-
self.save()
262274
self._scheduler.start(self._job_id)
263-
self.state = 'up'
264-
self.save()
265275

276+
@transition(field=state,
277+
source=[INITIALIZED, CREATED, UP, DOWN],
278+
target=UP,
279+
crashed=DOWN)
266280
def deploy(self, release):
267281
old_job_id = self._job_id
268-
self.state = 'deploying'
269-
self.save()
270282
# update release
271283
self.release = release
272284
self.save()
@@ -278,30 +290,24 @@ def deploy(self, release):
278290
self._scheduler.start(new_job_id)
279291
# destroy old container
280292
self._scheduler.destroy(old_job_id)
281-
self.state = 'up'
282-
self.save()
283293

294+
@transition(field=state, source=UP, target=DOWN)
284295
def stop(self):
285-
self.state = 'stopping'
286-
self.save()
287296
self._scheduler.stop(self._job_id)
288-
self.state = 'stopped'
289-
self.save()
290297

298+
@transition(field=state,
299+
source=[INITIALIZED, CREATED, UP, DOWN],
300+
target=DESTROYED)
291301
def destroy(self):
292-
self.state = 'destroying'
293-
self.save()
294302
# TODO: add check for active connections before killing
295303
self._scheduler.destroy(self._job_id)
296-
self.state = 'destroyed'
297-
self.save()
298304

305+
@transition(field=state,
306+
source=[INITIALIZED, CREATED, DESTROYED],
307+
target=DESTROYED)
299308
def run(self, command):
300-
self.state = 'running'
301-
self.save()
309+
"""Run a one-off command"""
302310
rc, output = self._scheduler.run(self._job_id, self.release.image, command)
303-
self.state = 'completed'
304-
self.save()
305311
return rc, output
306312

307313

‎api/tasks.py‎

Lines changed: 10 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,8 @@ def deploy_release(app, release):
3030
threads = []
3131
for c in containers:
3232
threads.append(threading.Thread(target=c.deploy, args=(release,)))
33-
try:
34-
[t.start() for t in threads]
35-
[t.join() for t in threads]
36-
except Exception:
37-
for c in containers:
38-
c.state = 'error'
39-
c.save()
40-
raise
33+
[t.start() for t in threads]
34+
[t.join() for t in threads]
4135

4236

4337
@task
@@ -47,16 +41,10 @@ def start_containers(containers):
4741
for c in containers:
4842
create_threads.append(threading.Thread(target=c.create))
4943
start_threads.append(threading.Thread(target=c.start))
50-
try:
51-
[t.start() for t in create_threads]
52-
[t.join() for t in create_threads]
53-
[t.start() for t in start_threads]
54-
[t.join() for t in start_threads]
55-
except Exception:
56-
for c in containers:
57-
c.state = 'error'
58-
c.save()
59-
raise
44+
[t.start() for t in create_threads]
45+
[t.join() for t in create_threads]
46+
[t.start() for t in start_threads]
47+
[t.join() for t in start_threads]
6048

6149

6250
@task
@@ -66,16 +54,10 @@ def stop_containers(containers):
6654
for c in containers:
6755
destroy_threads.append(threading.Thread(target=c.destroy))
6856
delete_threads.append(threading.Thread(target=c.delete))
69-
try:
70-
[t.start() for t in destroy_threads]
71-
[t.join() for t in destroy_threads]
72-
[t.start() for t in delete_threads]
73-
[t.join() for t in delete_threads]
74-
except Exception:
75-
for c in containers:
76-
c.state = 'error'
77-
c.save()
78-
raise
57+
[t.start() for t in destroy_threads]
58+
[t.join() for t in destroy_threads]
59+
[t.start() for t in delete_threads]
60+
[t.join() for t in delete_threads]
7961

8062

8163
@task

‎api/tests/test_container.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ def test_container(self):
4343
release=App.objects.get(id=app_id).release_set.latest(),
4444
type='web',
4545
num=1)
46-
self.assertEqual(c.state, 'initializing')
46+
self.assertEqual(c.state, 'initialized')
4747
c.create()
4848
self.assertEqual(c.state, 'created')
4949
c.start()

‎deis/settings.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@
137137
# Third-party apps
138138
'allauth',
139139
'allauth.account',
140+
'django_fsm',
140141
'guardian',
141142
'json_field',
142143
'gunicorn',

‎requirements.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ boto==2.23.0
77
celery==3.1.8
88
Django==1.6.2
99
django-allauth==0.15.0
10+
# HACK: use bacongobbler's feature branch until
11+
# https://github.com/kmmbvnr/django-fsm/pull/31 is merged
12+
git+https://github.com/bacongobbler/django-fsm@add-exception-handling
1013
django-guardian==1.1.1
1114
django-json-field==0.5.5
1215
django-yamlfield==0.5

0 commit comments

Comments
 (0)