Skip to content

Commit 36544e7

Browse files
author
Gabriel Monroy
committed
Merge pull request #967 from deis/dockerfile-workflow
Improve Dockerfile and Procfile workflows
2 parents e487fe2 + 2dee62c commit 36544e7

19 files changed

Lines changed: 967 additions & 311 deletions

‎builder/templates/builder‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ if __name__ == '__main__':
5656
if rc != 0:
5757
raise Exception('Could not extract git archive')
5858
dockerfile = os.path.join(temp_dir, 'Dockerfile')
59+
procfile = os.path.join(temp_dir, 'Procfile')
5960
# pull config to be used during build
6061
body = {}
6162
body['receive_user'], body['receive_repo'] = user, app
@@ -113,6 +114,28 @@ if __name__ == '__main__':
113114
body['receive_user'] = user
114115
body['receive_repo'] = app
115116
body['image'] = target_image
117+
# use sha of master
118+
with open(os.path.join(repo_dir, 'refs/heads/master')) as f:
119+
body['sha'] = f.read().strip('\n')
120+
# extract the user-defined Procfile and any default_process_types
121+
procfile_dict = {}
122+
p = subprocess.Popen('tar --to-stdout -xzf {temp_dir}/slug.tgz ./.release'.format(**locals()), shell=True, cwd=temp_dir,
123+
stdout=subprocess.PIPE, stderr=open(os.devnull, 'w'))
124+
rc = p.wait()
125+
if rc == 0:
126+
stdout = p.stdout.read()
127+
default_process_types = yaml.safe_load(stdout).get('default_process_types', {})
128+
procfile_dict.update(default_process_types)
129+
if os.path.exists(procfile):
130+
with open(procfile) as f:
131+
raw_procfile = f.read()
132+
procfile_dict.update(yaml.safe_load(raw_procfile))
133+
if procfile_dict:
134+
body['procfile'] = json.dumps(procfile_dict)
135+
# extract Dockerfile
136+
if os.path.exists(dockerfile):
137+
with open(dockerfile) as f:
138+
body['dockerfile'] = f.read()
116139
# trigger build hook
117140
sys.stdout.write('\n Launching... ')
118141
sys.stdout.flush()

‎controller/api/migrations/0010_auto__add_field_build_sha__add_field_build_procfile__add_field_build_d.py‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
# -*- coding: utf-8 -*-
2+
from south.utils import datetime_utils as datetime
3+
from south.db import db
4+
from south.v2 import SchemaMigration
5+
from django.db import models
6+
7+
8+
class Migration(SchemaMigration):
9+
10+
def forwards(self, orm):
11+
# Adding field 'Build.sha'
12+
db.add_column(u'api_build', 'sha',
13+
self.gf('django.db.models.fields.CharField')(default='', max_length=40, blank=True),
14+
keep_default=False)
15+
16+
# Adding field 'Build.procfile'
17+
db.add_column(u'api_build', 'procfile',
18+
self.gf('json_field.fields.JSONField')(default=u'{}', blank=True),
19+
keep_default=False)
20+
21+
# Adding field 'Build.dockerfile'
22+
db.add_column(u'api_build', 'dockerfile',
23+
self.gf('django.db.models.fields.TextField')(default='', blank=True),
24+
keep_default=False)
25+
26+
27+
def backwards(self, orm):
28+
# Deleting field 'Build.sha'
29+
db.delete_column(u'api_build', 'sha')
30+
31+
# Deleting field 'Build.procfile'
32+
db.delete_column(u'api_build', 'procfile')
33+
34+
# Deleting field 'Build.dockerfile'
35+
db.delete_column(u'api_build', 'dockerfile')
36+
37+
38+
models = {
39+
u'api.app': {
40+
'Meta': {'object_name': 'App'},
41+
'cluster': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Cluster']"}),
42+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
43+
'id': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '64'}),
44+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
45+
'structure': ('json_field.fields.JSONField', [], {'default': "u'{}'", 'blank': 'True'}),
46+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
47+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
48+
},
49+
u'api.build': {
50+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Build'},
51+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
52+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
53+
'dockerfile': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
54+
'image': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
55+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
56+
'procfile': ('json_field.fields.JSONField', [], {'default': "u'{}'", 'blank': 'True'}),
57+
'sha': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'}),
58+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
59+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
60+
},
61+
u'api.cluster': {
62+
'Meta': {'object_name': 'Cluster'},
63+
'auth': ('django.db.models.fields.TextField', [], {}),
64+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
65+
'domain': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
66+
'hosts': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
67+
'id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
68+
'options': ('json_field.fields.JSONField', [], {'default': "u'{}'", 'blank': 'True'}),
69+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
70+
'type': ('django.db.models.fields.CharField', [], {'default': "u'coreos'", 'max_length': '16'}),
71+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
72+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
73+
},
74+
u'api.config': {
75+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Config'},
76+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
77+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
78+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
79+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
80+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'}),
81+
'values': ('json_field.fields.JSONField', [], {'default': "u'{}'", 'blank': 'True'})
82+
},
83+
u'api.container': {
84+
'Meta': {'ordering': "[u'created']", 'object_name': 'Container'},
85+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
86+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
87+
'num': ('django.db.models.fields.PositiveIntegerField', [], {}),
88+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
89+
'release': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Release']"}),
90+
'state': ('django_fsm.FSMField', [], {'default': "u'initialized'", 'max_length': '50'}),
91+
'type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
92+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
93+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
94+
},
95+
u'api.key': {
96+
'Meta': {'unique_together': "((u'owner', u'id'),)", 'object_name': 'Key'},
97+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
98+
'id': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
99+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
100+
'public': ('django.db.models.fields.TextField', [], {'unique': 'True'}),
101+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
102+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
103+
},
104+
u'api.push': {
105+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Push'},
106+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
107+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
108+
'fingerprint': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
109+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
110+
'receive_repo': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
111+
'receive_user': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
112+
'sha': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
113+
'ssh_connection': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
114+
'ssh_original_command': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
115+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
116+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
117+
},
118+
u'api.release': {
119+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'version'),)", 'object_name': 'Release'},
120+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
121+
'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Build']"}),
122+
'config': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Config']"}),
123+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
124+
'image': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
125+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
126+
'summary': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
127+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
128+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'}),
129+
'version': ('django.db.models.fields.PositiveIntegerField', [], {})
130+
},
131+
u'auth.group': {
132+
'Meta': {'object_name': 'Group'},
133+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
134+
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
135+
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
136+
},
137+
u'auth.permission': {
138+
'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
139+
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
140+
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
141+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
142+
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
143+
},
144+
u'auth.user': {
145+
'Meta': {'object_name': 'User'},
146+
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
147+
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
148+
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
149+
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
150+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
151+
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
152+
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
153+
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
154+
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
155+
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
156+
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
157+
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
158+
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
159+
},
160+
u'contenttypes.contenttype': {
161+
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
162+
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
163+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
164+
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
165+
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
166+
}
167+
}
168+
169+
complete_apps = ['api']
170+

‎controller/api/models.py‎

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from celery.canvas import group
1515
from django.conf import settings
1616
from django.contrib.auth.models import User
17-
from django.db import models
17+
from django.db import models, connections
1818
from django.db.models import Max
1919
from django.db.models.signals import post_delete
2020
from django.db.models.signals import post_save
@@ -36,6 +36,20 @@ def log_event(app, msg, level=logging.INFO):
3636
logger.log(level, msg)
3737

3838

39+
def close_db_connections(func, *args, **kwargs):
40+
"""
41+
Decorator to close db connections during threaded execution
42+
43+
Note this is necessary to work around:
44+
https://code.djangoproject.com/ticket/22420
45+
"""
46+
def _inner(*args, **kwargs):
47+
func(*args, **kwargs)
48+
for conn in connections.all():
49+
conn.close()
50+
return _inner
51+
52+
3953
class AuditedModel(models.Model):
4054
"""Add created and updated fields to a model."""
4155

@@ -127,16 +141,39 @@ def delete(self, *args, **kwargs):
127141
c.destroy()
128142
return super(App, self).delete(*args, **kwargs)
129143

130-
def deploy(self, release):
144+
def deploy(self, release, initial=False):
131145
tasks.deploy_release.delay(self, release).get()
146+
if initial:
147+
# if there is no SHA, assume a docker image is being promoted
148+
if not release.build.sha:
149+
self.structure = {'cmd': 1}
150+
# if a dockerfile exists without a procfile, assume docker workflow
151+
elif release.build.dockerfile and not release.build.procfile:
152+
self.structure = {'cmd': 1}
153+
# if a procfile exists without a web entry, assume docker workflow
154+
elif release.build.procfile and not 'web' in release.build.procfile:
155+
self.structure = {'cmd': 1}
156+
# default to heroku workflow
157+
else:
158+
self.structure = {'web': 1}
159+
self.save()
160+
self.scale()
132161

133162
def destroy(self, *args, **kwargs):
134163
return self.delete(*args, **kwargs)
135164

136-
def scale(self, **kwargs):
165+
def scale(self, **kwargs): # noqa
137166
"""Scale containers up or down to match requested."""
138167
requested_containers = self.structure.copy()
139168
release = self.release_set.latest()
169+
# test for available process types
170+
available_process_types = release.build.procfile or {}
171+
for container_type in requested_containers.keys():
172+
if container_type == 'cmd':
173+
continue # allow docker cmd types in case we don't have the image source
174+
if not container_type in available_process_types:
175+
raise EnvironmentError(
176+
'Container type {} does not exist in application'.format(container_type))
140177
msg = 'Containers scaled ' + ' '.join(
141178
"{}={}".format(k, v) for k, v in requested_containers.items())
142179
# iterate and scale by container type (web, worker, etc)
@@ -146,7 +183,7 @@ def scale(self, **kwargs):
146183
containers = list(self.container_set.filter(type=container_type).order_by('created'))
147184
# increment new container nums off the most recent container
148185
results = self.container_set.filter(type=container_type).aggregate(Max('num'))
149-
container_num = results.get('num__max') or 0 + 1
186+
container_num = (results.get('num__max') or 0) + 1
150187
requested = requested_containers.pop(container_type)
151188
diff = requested - len(containers)
152189
if diff == 0:
@@ -267,18 +304,21 @@ def _get_command(self):
267304

268305
_command = property(_get_command)
269306

307+
@close_db_connections
270308
@transition(field=state, source=INITIALIZED, target=CREATED)
271309
def create(self):
272310
image = self.release.image
273311
c_type = self.type
274312
self._scheduler.create(self._job_id, image, self._command.format(**locals()))
275313

314+
@close_db_connections
276315
@transition(field=state,
277316
source=[CREATED, UP, DOWN],
278317
target=UP, crashed=DOWN)
279318
def start(self):
280319
self._scheduler.start(self._job_id)
281320

321+
@close_db_connections
282322
@transition(field=state,
283323
source=[INITIALIZED, CREATED, UP, DOWN],
284324
target=UP,
@@ -297,10 +337,12 @@ def deploy(self, release):
297337
# destroy old container
298338
self._scheduler.destroy(old_job_id)
299339

340+
@close_db_connections
300341
@transition(field=state, source=UP, target=DOWN)
301342
def stop(self):
302343
self._scheduler.stop(self._job_id)
303344

345+
@close_db_connections
304346
@transition(field=state,
305347
source=[INITIALIZED, CREATED, UP, DOWN],
306348
target=DESTROYED)
@@ -352,6 +394,11 @@ class Build(UuidAuditedModel):
352394
app = models.ForeignKey('App')
353395
image = models.CharField(max_length=256)
354396

397+
# optional fields populated by builder
398+
sha = models.CharField(max_length=40, blank=True)
399+
procfile = JSONField(default='{}', blank=True)
400+
dockerfile = models.TextField(blank=True)
401+
355402
class Meta:
356403
get_latest_by = 'created'
357404
ordering = ['-created']
@@ -455,7 +502,10 @@ def save(self, *args, **kwargs):
455502
old_build = prev_release.build if prev_release else None
456503
# if the build changed, log it and who pushed it
457504
if self.build != old_build:
458-
self.summary += "{} deployed {}".format(self.build.owner, self.build.image)
505+
if self.build.sha:
506+
self.summary += "{} deployed {}".format(self.build.owner, self.build.sha[:7])
507+
else:
508+
self.summary += "{} deployed {}".format(self.build.owner, self.build.image)
459509
# compare this config to the previous config
460510
old_config = prev_release.config if prev_release else None
461511
# if the config data changed, log the dict diff

0 commit comments

Comments
 (0)