Skip to content

Commit ac956d1

Browse files
Gabriel MonroyMatthew Fisher
Gabriel Monroy
authored and
Matthew Fisher
committed
refactor(registry): new registry package with mock and private implementations
 - mock docker registry used for unit testing - private module used for adding layers to private registry - add registry.image field that includes build+config as a tagged image - added confd settings for a required private registry_url
1 parent a20798a commit ac956d1

10 files changed

Lines changed: 216 additions & 17 deletions

File tree

‎controller/api/migrations/0008_auto__add_field_release_image.py‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
# -*- coding: utf-8 -*-
2+
from south.utils import datetime_utils as datetime
3+
from south.db import db
4+
from south.v2 import SchemaMigration
5+
from django.db import models
6+
7+
8+
class Migration(SchemaMigration):
9+
10+
def forwards(self, orm):
11+
# Adding field 'Release.image'
12+
db.add_column(u'api_release', 'image',
13+
self.gf('django.db.models.fields.CharField')(default='hi', max_length=256),
14+
keep_default=False)
15+
16+
17+
def backwards(self, orm):
18+
# Deleting field 'Release.image'
19+
db.delete_column(u'api_release', 'image')
20+
21+
22+
models = {
23+
u'api.app': {
24+
'Meta': {'object_name': 'App'},
25+
'cluster': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Cluster']"}),
26+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
27+
'id': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '64'}),
28+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
29+
'structure': ('json_field.fields.JSONField', [], {'default': "u'{}'", 'blank': 'True'}),
30+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
31+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
32+
},
33+
u'api.build': {
34+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Build'},
35+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
36+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
37+
'image': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
38+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
39+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
40+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
41+
},
42+
u'api.cluster': {
43+
'Meta': {'object_name': 'Cluster'},
44+
'auth': ('django.db.models.fields.TextField', [], {}),
45+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
46+
'domain': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
47+
'hosts': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
48+
'id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
49+
'options': ('json_field.fields.JSONField', [], {'default': "u'{}'", 'blank': 'True'}),
50+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
51+
'type': ('django.db.models.fields.CharField', [], {'max_length': '16'}),
52+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
53+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
54+
},
55+
u'api.config': {
56+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Config'},
57+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
58+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
59+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
60+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
61+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'}),
62+
'values': ('json_field.fields.JSONField', [], {'default': "u'{}'", 'blank': 'True'})
63+
},
64+
u'api.container': {
65+
'Meta': {'ordering': "[u'created']", 'object_name': 'Container'},
66+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
67+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
68+
'num': ('django.db.models.fields.PositiveIntegerField', [], {}),
69+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
70+
'release': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Release']"}),
71+
'state': ('django.db.models.fields.CharField', [], {'default': "u'initializing'", 'max_length': '64'}),
72+
'type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
73+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
74+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
75+
},
76+
u'api.key': {
77+
'Meta': {'unique_together': "((u'owner', u'id'),)", 'object_name': 'Key'},
78+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
79+
'id': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
80+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
81+
'public': ('django.db.models.fields.TextField', [], {'unique': 'True'}),
82+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
83+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
84+
},
85+
u'api.push': {
86+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'uuid'),)", 'object_name': 'Push'},
87+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
88+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
89+
'fingerprint': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
90+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
91+
'receive_repo': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
92+
'receive_user': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
93+
'sha': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
94+
'ssh_connection': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
95+
'ssh_original_command': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
96+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
97+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'})
98+
},
99+
u'api.release': {
100+
'Meta': {'ordering': "[u'-created']", 'unique_together': "((u'app', u'version'),)", 'object_name': 'Release'},
101+
'app': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.App']"}),
102+
'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Build']"}),
103+
'config': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['api.Config']"}),
104+
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
105+
'image': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
106+
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
107+
'summary': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
108+
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
109+
'uuid': ('api.fields.UuidField', [], {'unique': 'True', 'max_length': '32', 'primary_key': 'True'}),
110+
'version': ('django.db.models.fields.PositiveIntegerField', [], {})
111+
},
112+
u'auth.group': {
113+
'Meta': {'object_name': 'Group'},
114+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
115+
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
116+
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
117+
},
118+
u'auth.permission': {
119+
'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
120+
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
121+
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
122+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
123+
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
124+
},
125+
u'auth.user': {
126+
'Meta': {'object_name': 'User'},
127+
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
128+
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
129+
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
130+
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
131+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
132+
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
133+
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
134+
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
135+
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
136+
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
137+
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
138+
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
139+
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
140+
},
141+
u'contenttypes.contenttype': {
142+
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
143+
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
144+
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
145+
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
146+
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
147+
}
148+
}
149+
150+
complete_apps = ['api']

‎controller/api/models.py‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
from json_field.fields import JSONField
2222

2323
from api import fields, tasks
24-
from docker import publish_release
24+
from registry import publish_release
2525
from utils import dict_diff, fingerprint
2626

2727

@@ -183,7 +183,7 @@ def run(self, command):
183183
raise EnvironmentError('No build exists, please run `git push deis master` first')
184184
# prepare ssh command
185185
version = release.version
186-
image = release.build.image + ":v{}".format(release.version)
186+
image = release.image
187187
docker_args = ' '.join(['-a', 'stdout', '-a', 'stderr', '-rm', image])
188188
env_args = ' '.join(["-e '{k}={v}'".format(**locals())
189189
for k, v in release.config.values.items()])
@@ -249,7 +249,7 @@ def _get_command(self):
249249
_command = property(_get_command)
250250

251251
def create(self):
252-
image = self.release.build.image
252+
image = self.release.image
253253
c_type = self.type
254254
self._scheduler.create(self._job_id, image, self._command.format(**locals()))
255255
self.state = 'created'
@@ -271,8 +271,9 @@ def deploy(self, release):
271271
self.save()
272272
# deploy new container
273273
new_job_id = self._job_id
274-
image = self.release.build.image
275-
self._scheduler.create(new_job_id, image, 'docker run {image}'.format(**locals()))
274+
image = self.release.image
275+
c_type = self.type
276+
self._scheduler.create(new_job_id, image, self._command.format(**locals()))
276277
self._scheduler.start(new_job_id)
277278
# destroy old container
278279
self._scheduler.destroy(old_job_id)
@@ -374,6 +375,8 @@ class Release(UuidAuditedModel):
374375

375376
config = models.ForeignKey('Config')
376377
build = models.ForeignKey('Build')
378+
# NOTE: image contains combined build + config, ready to run
379+
image = models.CharField(max_length=256)
377380

378381
class Meta:
379382
get_latest_by = 'created'
@@ -394,16 +397,17 @@ def new(self, user, config=None, build=None):
394397
config = self.config
395398
if not build:
396399
build = self.build
397-
# create new release and auto-increment version
400+
# prepare release tag
398401
new_version = self.version + 1
402+
tag = 'v{}'.format(new_version)
403+
image = build.image + ':{tag}'.format(**locals())
404+
# create new release and auto-increment version
399405
release = Release.objects.create(
400406
owner=user, app=self.app, config=config,
401-
build=build, version=new_version)
407+
build=build, version=new_version, image=image)
402408
# publish release to registry as new docker image
403-
if settings.REGISTRY_URL:
404-
repository_path = "{}/{}".format(user.username, self.app.id)
405-
tag = 'v{}'.format(new_version)
406-
publish_release(repository_path, config.values, tag)
409+
repository_path = "{}/{}".format(user.username, self.app.id)
410+
publish_release(repository_path, config.values, tag)
407411
return release
408412

409413
def previous(self):

‎controller/api/tests/test_config.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,28 @@ def test_config(self):
9494
self.assertEqual(self.client.delete(url).status_code, 405)
9595
return config5
9696

97+
def test_config_set_same_key(self):
98+
"""
99+
Test that config sets on the same key function properly
100+
"""
101+
url = '/api/apps'
102+
body = {'cluster': 'autotest'}
103+
response = self.client.post(url, json.dumps(body), content_type='application/json')
104+
self.assertEqual(response.status_code, 201)
105+
app_id = response.data['id']
106+
url = "/api/apps/{app_id}/config".format(**locals())
107+
# set an initial config value
108+
body = {'values': json.dumps({'PORT': '5000'})}
109+
response = self.client.post(url, json.dumps(body), content_type='application/json')
110+
self.assertEqual(response.status_code, 201)
111+
self.assertIn('PORT', json.loads(response.data['values']))
112+
# reset same config value
113+
body = {'values': json.dumps({'PORT': '5001'})}
114+
response = self.client.post(url, json.dumps(body), content_type='application/json')
115+
self.assertEqual(response.status_code, 201)
116+
self.assertIn('PORT', json.loads(response.data['values']))
117+
self.assertEqual(json.loads(response.data['values'])['PORT'], '5001')
118+
97119
def test_config_str(self):
98120
"""Test the text representation of a node."""
99121
config5 = self.test_config()

‎controller/api/views.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@
2222
from rest_framework.generics import get_object_or_404
2323
from rest_framework.response import Response
2424

25-
from api import docker, models, serializers
25+
from api import models, serializers
26+
from registry import publish_release
2627
from .exceptions import UserRegistrationException
2728

2829
from django.conf import settings
@@ -413,6 +414,7 @@ def get_object(self, *args, **kwargs):
413414
"""Get Release by version always."""
414415
return self.get_queryset(**kwargs).get(version=self.kwargs['version'])
415416

417+
# TODO: move logic into model
416418
def rollback(self, request, *args, **kwargs):
417419
"""
418420
Create a new release as a copy of the state of the compiled slug and
@@ -430,11 +432,9 @@ def rollback(self, request, *args, **kwargs):
430432
build=prev.build, config=prev.config,
431433
summary=summary)
432434
# publish release to registry as new docker image
433-
if settings.REGISTRY_URL:
434-
repository_path = "{}/{}".format(app.owner.username, app.id)
435-
tag = 'v{}'.format(last_version + 1)
436-
docker.publish_release(repository_path, prev.config.values, tag)
437-
app.converge()
435+
repository_path = "{}/{}".format(app.owner.username, app.id)
436+
tag = 'v{}'.format(last_version + 1)
437+
publish_release(repository_path, prev.config.values, tag)
438438
msg = "Rolled back to v{}".format(version)
439439
return Response(msg, status=status.HTTP_201_CREATED)
440440

‎controller/conf.d/confd_settings.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ keys = [
88
"/deis/controller",
99
"/deis/cache",
1010
"/deis/database",
11+
"/deis/registry",
1112
]
1213
check_cmd = "test -e {{ .src }}"
1314
reload_cmd = "/app/bin/reload"

‎controller/deis/settings.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,7 @@
276276
BUILDER_KEY = os.environ.get('DEIS_BUILDER_KEY', 'CHANGEME_sapm$s%upvsw5l_zuy_&29rkywd^78ff(qi')
277277

278278
# registry settings
279+
REGISTRY_MODULE = 'registry.mock'
279280
REGISTRY_URL = os.environ.get('DEIS_REGISTRY_URL', None)
280281

281282
# check if we can register users with `deis register`

‎controller/registry/__init__.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import importlib
2+
3+
from deis import settings
4+
5+
# import the registry module specified in settings
6+
_registry_module = importlib.import_module(settings.REGISTRY_MODULE)
7+
8+
# expose the publish_release method publicly
9+
publish_release = _registry_module.publish_release

‎controller/registry/mock.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
2+
def publish_release(repository_path, config, tag):
3+
"""
4+
Publish a new release as a Docker image
5+
6+
This is a mock implementation used for unit tests
7+
"""
8+
return None

controller/api/docker.py renamed to controller/registry/private.py

File renamed without changes.

‎controller/templates/confd_settings.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22
SECRET_KEY = '{{ .deis_controller_secretKey }}'
33
BUILDER_KEY = '{{ .deis_controller_builderKey }}'
44

5+
# use the private registry module
6+
REGISTRY_MODULE = 'registry.private'
7+
REGISTRY_URL = '{{ .deis_registry_protocol }}://{{ .deis_registry_host }}:{{ .deis_registry_port }}' # noqa
8+
59
# default to sqlite3, but allow postgresql config through envvars
610
DATABASES = {
711
'default': {

0 commit comments

Comments
 (0)