Skip to content

Commit cd6a38e

Browse files
author
Matthew Fisher
committed
feat(controller): add container state via FSM
This makes a container's state directly managed via a Finite State Machine (see http://en.wikipedia.org/wiki/Finite-state_machine). With this new behaviour, containers can be in one of 5 different states: 1) up 2) down 3) initialized 4) created 5) destroyed A diagram that shows the container's transitions as it goes from state to state would be nice, but is not necessary for this commit. I've noted this as a TODO feature in the documentation. I have also set the container's FSM to protected, so that nobody can directly change the FSM's state. Only a call to the model's functions that have a @transition decorator will be able to change the state. fixes #699
1 parent 231ede3 commit cd6a38e

8 files changed

Lines changed: 226 additions & 54 deletions

File tree

‎controller/Dockerfile‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,27 @@ RUN apt-get update && \
99
RUN wget -qO- https://raw.github.com/pypa/pip/1.5.4/contrib/get-pip.py | python -
1010

1111
# install requirements before ADD to cache layer and speed build
12-
RUN pip install boto==2.23.0 celery==3.1.8 Django==1.6.2 django-allauth==0.15.0 django-guardian==1.1.1 django-json-field==0.5.5 django-yamlfield==0.5 djangorestframework==2.3.12 dop==0.1.4 gevent==1.0 gunicorn==18.0 paramiko==1.12.1 psycopg2==2.5.2 pycrypto==2.6.1 python-etcd==0.3.0 pyrax==1.6.2 PyYAML==3.10 redis==2.8.0 static==1.0.2 South==0.8.4
12+
RUN pip install boto==2.23.0 \
13+
celery==3.1.8 \
14+
Django==1.6.2 \
15+
django-allauth==0.15.0 \
16+
git+https://github.com/bacongobbler/django-fsm@add-exception-handling \
17+
django-guardian==1.1.1 \
18+
django-json-field==0.5.5 \
19+
django-yamlfield==0.5 \
20+
djangorestframework==2.3.12 \
21+
dop==0.1.6 \
22+
gevent==1.0 \
23+
gunicorn==18.0 \
24+
paramiko==1.12.1 \
25+
psycopg2==2.5.2 \
26+
pycrypto==2.6.1 \
27+
python-etcd==0.3.0 \
28+
pyrax==1.6.2 \
29+
PyYAML==3.10 \
30+
redis==2.8.0 \
31+
static==1.0.2 \
32+
South==0.8.4
1333

1434
# install openssh-client for temporary fleetctl wrapper
1535
RUN apt-get install -yq openssh-client

‎controller/api/migrations/0009_auto__chg_field_container_state.py‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
# -*- coding: utf-8 -*-
2+
from south.utils import datetime_utils as datetime
3+
from south.db import db
4+
from south.v2 import SchemaMigration
5+
from django.db import models
6+
7+
8+
class Migration(SchemaMigration):
9+
10+
def forwards(self, orm):
11+
12+
# Changing field 'Container.state'
13+
db.alter_column(u'api_container', 'state', self.gf('django_fsm.FSMField')(max_length=50))
14+
15+
def backwards(self, orm):
16+
17+
# Changing field 'Container.state'
18+
db.alter_column(u'api_container', 'state', self.gf('django.db.models.fields.CharField')(max_length=64))
19+
20+
models = {
21+
u'api.app': {
22+
'Meta': {'object_name': 'App'},
23+
'cluster': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Cluster']"}),
24+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
25+
'id': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '64'}),
26+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
27+
'structure': ('json_field.fields.JSONField', [], {'default': "u'{}'", 'blank': 'True'}),
28+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
29+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
30+
},
31+
u'api.build': {
32+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Build'},
33+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
34+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
35+
'image': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
36+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
37+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
38+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
39+
},
40+
u'api.cluster': {
41+
'Meta': {'object_name': 'Cluster'},
42+
'auth': ('django.db.models.fields.TextField', [], {}),
43+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
44+
'domain': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
45+
'hosts': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
46+
'id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
47+
'options': ('json_field.fields.JSONField', [], {'default': "u'{}'", 'blank': 'True'}),
48+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
49+
'type': ('django.db.models.fields.CharField', [], {'max_length': '16'}),
50+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
51+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
52+
},
53+
u'api.config': {
54+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Config'},
55+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
56+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
57+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
58+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
59+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'}),
60+
'values': ('json_field.fields.JSONField', [], {'default': "u'{}'", 'blank': 'True'})
61+
},
62+
u'api.container': {
63+
'Meta': {'ordering': "[u'created']", 'object_name': 'Container'},
64+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
65+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
66+
'num': ('django.db.models.fields.PositiveIntegerField', [], {}),
67+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
68+
'release': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Release']"}),
69+
'state': ('django_fsm.FSMField', [], {'default': "u'initialized'", 'max_length': '50'}),
70+
'type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
71+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
72+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
73+
},
74+
u'api.key': {
75+
'Meta': {'unique_together': "((u'owner', u'id'),)", 'object_name': 'Key'},
76+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
77+
'id': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
78+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
79+
'public': ('django.db.models.fields.TextField', [], {'unique': 'True'}),
80+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
81+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
82+
},
83+
u'api.push': {
84+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Push'},
85+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
86+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
87+
'fingerprint': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
88+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
89+
'receive_repo': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
90+
'receive_user': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
91+
'sha': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
92+
'ssh_connection': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
93+
'ssh_original_command': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
94+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
95+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
96+
},
97+
u'api.release': {
98+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'version'),)", 'object_name': 'Release'},
99+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
100+
'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Build']"}),
101+
'config': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Config']"}),
102+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
103+
'image': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
104+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
105+
'summary': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
106+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
107+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'}),
108+
'version': ('django.db.models.fields.PositiveIntegerField', [], {})
109+
},
110+
u'auth.group': {
111+
'Meta': {'object_name': 'Group'},
112+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
113+
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
114+
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
115+
},
116+
u'auth.permission': {
117+
'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
118+
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
119+
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
120+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
121+
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
122+
},
123+
u'auth.user': {
124+
'Meta': {'object_name': 'User'},
125+
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
126+
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
127+
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
128+
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
129+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
130+
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
131+
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
132+
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
133+
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
134+
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
135+
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
136+
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
137+
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
138+
},
139+
u'contenttypes.contenttype': {
140+
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
141+
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
142+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
143+
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
144+
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
145+
}
146+
}
147+
148+
complete_apps = ['api']

‎controller/api/models.py‎

Lines changed: 30 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from django.db.models.signals import post_delete
1919
from django.db.models.signals import post_save
2020
from django.utils.encoding import python_2_unicode_compatible
21+
from django_fsm import FSMField, transition
2122
from json_field.fields import JSONField
2223

2324
from api import fields, tasks
@@ -199,14 +200,25 @@ class Container(UuidAuditedModel):
199200
"""
200201
Docker container used to securely host an application process.
201202
"""
203+
INITIALIZED = 'initialized'
204+
CREATED = 'created'
205+
UP = 'up'
206+
DOWN = 'down'
207+
DESTROYED = 'destroyed'
208+
STATE_CHOICES = (
209+
(INITIALIZED, 'initialized'),
210+
(CREATED, 'created'),
211+
(UP, 'up'),
212+
(DOWN, 'down'),
213+
(DESTROYED, 'destroyed')
214+
)
202215

203216
owner = models.ForeignKey(settings.AUTH_USER_MODEL)
204217
app = models.ForeignKey('App')
205218
release = models.ForeignKey('Release')
206219
type = models.CharField(max_length=128, blank=True)
207220
num = models.PositiveIntegerField()
208-
# TODO: implement fsm
209-
state = models.CharField(max_length=64, default='initializing')
221+
state = FSMField(default=INITIALIZED, choices=STATE_CHOICES, protected=True)
210222

211223
def short_name(self):
212224
if self.type:
@@ -249,24 +261,24 @@ def _get_command(self):
249261

250262
_command = property(_get_command)
251263

264+
@transition(field=state, source=INITIALIZED, target=CREATED)
252265
def create(self):
253266
image = self.release.image
254267
c_type = self.type
255268
self._scheduler.create(self._job_id, image, self._command.format(**locals()))
256-
self.state = 'created'
257-
self.save()
258269

270+
@transition(field=state,
271+
source=[CREATED, UP, DOWN],
272+
target=UP, crashed=DOWN)
259273
def start(self):
260-
self.state = 'starting'
261-
self.save()
262274
self._scheduler.start(self._job_id)
263-
self.state = 'up'
264-
self.save()
265275

276+
@transition(field=state,
277+
source=[INITIALIZED, CREATED, UP, DOWN],
278+
target=UP,
279+
crashed=DOWN)
266280
def deploy(self, release):
267281
old_job_id = self._job_id
268-
self.state = 'deploying'
269-
self.save()
270282
# update release
271283
self.release = release
272284
self.save()
@@ -278,30 +290,24 @@ def deploy(self, release):
278290
self._scheduler.start(new_job_id)
279291
# destroy old container
280292
self._scheduler.destroy(old_job_id)
281-
self.state = 'up'
282-
self.save()
283293

294+
@transition(field=state, source=UP, target=DOWN)
284295
def stop(self):
285-
self.state = 'stopping'
286-
self.save()
287296
self._scheduler.stop(self._job_id)
288-
self.state = 'stopped'
289-
self.save()
290297

298+
@transition(field=state,
299+
source=[INITIALIZED, CREATED, UP, DOWN],
300+
target=DESTROYED)
291301
def destroy(self):
292-
self.state = 'destroying'
293-
self.save()
294302
# TODO: add check for active connections before killing
295303
self._scheduler.destroy(self._job_id)
296-
self.state = 'destroyed'
297-
self.save()
298304

305+
@transition(field=state,
306+
source=[INITIALIZED, CREATED, DESTROYED],
307+
target=DESTROYED)
299308
def run(self, command):
300-
self.state = 'running'
301-
self.save()
309+
"""Run a one-off command"""
302310
rc, output = self._scheduler.run(self._job_id, self.release.image, command)
303-
self.state = 'completed'
304-
self.save()
305311
return rc, output
306312

307313

‎controller/api/tasks.py‎

Lines changed: 10 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,8 @@ def deploy_release(app, release):
3030
threads = []
3131
for c in containers:
3232
threads.append(threading.Thread(target=c.deploy, args=(release,)))
33-
try:
34-
[t.start() for t in threads]
35-
[t.join() for t in threads]
36-
except Exception:
37-
for c in containers:
38-
c.state = 'error'
39-
c.save()
40-
raise
33+
[t.start() for t in threads]
34+
[t.join() for t in threads]
4135

4236

4337
@task
@@ -47,16 +41,10 @@ def start_containers(containers):
4741
for c in containers:
4842
create_threads.append(threading.Thread(target=c.create))
4943
start_threads.append(threading.Thread(target=c.start))
50-
try:
51-
[t.start() for t in create_threads]
52-
[t.join() for t in create_threads]
53-
[t.start() for t in start_threads]
54-
[t.join() for t in start_threads]
55-
except Exception:
56-
for c in containers:
57-
c.state = 'error'
58-
c.save()
59-
raise
44+
[t.start() for t in create_threads]
45+
[t.join() for t in create_threads]
46+
[t.start() for t in start_threads]
47+
[t.join() for t in start_threads]
6048

6149

6250
@task
@@ -66,16 +54,10 @@ def stop_containers(containers):
6654
for c in containers:
6755
destroy_threads.append(threading.Thread(target=c.destroy))
6856
delete_threads.append(threading.Thread(target=c.delete))
69-
try:
70-
[t.start() for t in destroy_threads]
71-
[t.join() for t in destroy_threads]
72-
[t.start() for t in delete_threads]
73-
[t.join() for t in delete_threads]
74-
except Exception:
75-
for c in containers:
76-
c.state = 'error'
77-
c.save()
78-
raise
57+
[t.start() for t in destroy_threads]
58+
[t.join() for t in destroy_threads]
59+
[t.start() for t in delete_threads]
60+
[t.join() for t in delete_threads]
7961

8062

8163
@task

‎controller/api/tests/test_container.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ def test_container(self):
4343
release=App.objects.get(id=app_id).release_set.latest(),
4444
type='web',
4545
num=1)
46-
self.assertEqual(c.state, 'initializing')
46+
self.assertEqual(c.state, 'initialized')
4747
c.create()
4848
self.assertEqual(c.state, 'created')
4949
c.start()

‎controller/deis/settings.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@
137137
# Third-party apps
138138
'allauth',
139139
'allauth.account',
140+
'django_fsm',
140141
'guardian',
141142
'json_field',
142143
'gunicorn',

‎controller/requirements.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ boto==2.23.0
77
celery==3.1.8
88
Django==1.6.2
99
django-allauth==0.15.0
10+
# HACK: use bacongobbler's feature branch until
11+
# https://github.com/kmmbvnr/django-fsm/pull/31 is merged
12+
git+https://github.com/bacongobbler/django-fsm@add-exception-handling
1013
django-guardian==1.1.1
1114
django-json-field==0.5.5
1215
django-yamlfield==0.5

‎docs/components/container.rst‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,16 @@ filesystem as a temporary scratchpad, but no files that are written are visible
1919
processes in any other container. Any files written to the ephemeral filesystem will be
2020
discarded the moment the container is either stopped or restarted.
2121

22+
Container States
23+
----------------
24+
25+
There are several states that a container can be in at any time. The states are:
26+
27+
1) initialized - the initial state of the container before it is created
28+
2) created - the container is built and ready for operation
29+
3) up - the container is running
30+
4) down - the container crashed or is stopped
31+
5) destroyed - the container has been destroyed
32+
33+
2234
.. _`Docker`: http://docker.io/

0 commit comments

Comments
 (0)