Skip to content

Commit 0998f3f

Browse files
duanhongyiduanhongyi
committed
feat(dryccfile): dryccfile support pipeline
1 parent ce32842 commit 0998f3f

8 files changed

Lines changed: 412 additions & 382 deletions

File tree

‎rootfs/api/models/build.py‎

Lines changed: 33 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ def type(self):
4545
@property
4646
def ptypes(self):
4747
if self.dryccfile:
48-
return list(self.dryccfile['deploy'].keys())
48+
return list([pipeline['ptype'] for pipeline in self.dryccfile['pipeline'].values()])
4949
if self.procfile:
5050
return list(self.procfile.keys())
5151
return [PTYPE_WEB]
@@ -64,14 +64,24 @@ def version(self):
6464
return 'git-{}'.format(self.sha) if self.source_based else 'latest'
6565

6666
def get_image(self, ptype, default_image=None):
67-
docker = self.dryccfile.get('build', {}).get('docker', {})
68-
if ptype in docker:
69-
if ptype == 'web':
70-
return self.image
71-
else:
72-
return f'{self.image}-{ptype}'
67+
pipeline = self.get_pipeline(ptype)
68+
if pipeline and 'build' in pipeline:
69+
return self.image if ptype == 'web' else f'{self.image}-{ptype}'
7370
return default_image if default_image else self.image
7471

72+
def get_pipeline(self, ptype):
73+
pipelines = self.get_pipelines([ptype])
74+
if len(pipelines) > 0:
75+
return pipelines[0]
76+
return {}
77+
78+
def get_pipelines(self, ptypes):
79+
results = []
80+
for pipeline in self.dryccfile.get('pipeline', {}).values():
81+
if pipeline['ptype'] in ptypes:
82+
results.append(pipeline)
83+
return results
84+
7585
def create_release(self, user, *args, **kwargs):
7686
latest_release = Release.latest(self.app)
7787
try:
@@ -110,38 +120,31 @@ def _get_or_create_config(self):
110120
"""
111121
dryccfile to config
112122
"""
113-
config_values, config_values_ref, config_healthcheck, changed_fields = (
114-
self.dryccfile.get('config', []), {}, {}, set())
115-
if 'config' in self.dryccfile:
123+
config_values, config_values_ref, changed_fields = [], {}, set()
124+
for group, envs in self.dryccfile.get('config', {}).items():
125+
for key, value in envs.items():
126+
config_values.append({"name": key, "group": group, "value": value})
116127
changed_fields.update(["values", "values_refs"])
117-
replace_values_ptypes, replace_healthcheck_ptypes = set(), set()
118-
for ptype, values in self.dryccfile.get('deploy', {}).items():
119-
if 'config' in values:
120-
for value in values.get('config', {}).get('env', []):
121-
value['ptype'] = ptype
122-
config_values.append(value)
123-
replace_values_ptypes.add(ptype)
124-
for config_ref in values.get('config', {}).get('ref', []):
125-
if ptype not in config_values_ref:
126-
config_values_ref[ptype] = [config_ref]
128+
129+
replace_values_ptypes = set()
130+
for pipeline in self.dryccfile.get('pipeline', {}).values():
131+
if 'env' in pipeline or 'config' in pipeline:
132+
for key, value in pipeline.get('env', {}).items():
133+
config_values.append({"name": key, "ptype": pipeline['ptype'], "value": value})
134+
for config_ref in pipeline.get('config', []):
135+
if pipeline['ptype'] not in config_values_ref:
136+
config_values_ref[pipeline['ptype']] = [config_ref]
127137
else:
128-
config_values_ref[ptype].append(config_ref)
129-
replace_values_ptypes.add(ptype)
138+
config_values_ref[pipeline['ptype']].append(config_ref)
139+
replace_values_ptypes.add(pipeline['ptype'])
130140
changed_fields.update(["values", "values_refs"])
131-
if 'healthcheck' in values:
132-
config_healthcheck[ptype] = values.get('healthcheck')
133-
changed_fields.add("healthcheck")
134-
replace_healthcheck_ptypes.add(ptype)
135141

136142
old_config = self.app.release_set.filter(failed=False).latest().config
137143
if not changed_fields:
138144
return old_config
139145
config = Config(
140-
owner=self.owner, app=self.app, values=config_values, values_refs=config_values_ref,
141-
healthcheck=config_healthcheck,
142-
)
146+
owner=self.owner, app=self.app, values=config_values, values_refs=config_values_ref)
143147
config.merge_field("values", old_config, replace_values_ptypes)
144148
config.merge_field("values_refs", old_config, replace_values_ptypes)
145-
config.merge_field("healthcheck", old_config, replace_healthcheck_ptypes)
146149
config.save(ignore_update_fields=changed_fields)
147150
return config

‎rootfs/api/models/release.py‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -67,15 +67,15 @@ def version_name(self):
6767
def get_runners(self, ptypes):
6868
results = []
6969
ptypes = self.ptypes if not ptypes else ptypes
70-
for ptype, run in self.build.dryccfile.get('run', {}).items():
71-
if ptype in ptypes:
72-
image = run.get('image', self.build.get_image(ptype))
70+
for pipeline in self.build.get_pipelines(ptypes):
71+
if 'run' in pipeline:
72+
image = pipeline['run'].get('image', self.build.get_image(pipeline['ptype']))
7373
results.append({
74-
'ptype': ptype,
74+
'ptype': pipeline['ptype'],
7575
'image': self.build.get_image(image, default_image=image),
76-
'args': run.get('args', []),
77-
'command': run.get('command', []),
78-
'timeout': run.get('timeout', settings.DRYCC_PILELINE_RUN_TIMEOUT),
76+
'args': pipeline['run'].get('args', []),
77+
'command': pipeline['run'].get('command', []),
78+
'timeout': pipeline['run'].get('timeout', settings.DRYCC_PILELINE_RUN_TIMEOUT),
7979
})
8080
return results
8181

@@ -93,21 +93,19 @@ def add_condition(self, **kwargs):
9393

9494
def get_deploy_image(self, ptype):
9595
"""
96-
In the deploy phase of dryccfile
9796
Return the kubernetes "container image" to be sent off to the scheduler.
9897
"""
99-
image = self.build.dryccfile.get('deploy', {}).get(ptype, {}).get(
98+
image = self.build.get_pipeline(ptype).get('deploy', {}).get(
10099
'image', self.build.get_image(ptype))
101100
return self.build.get_image(image, default_image=image)
102101

103102
def get_deploy_args(self, ptype):
104103
"""
105-
In the deploy phase of dryccfile
106104
Return the kubernetes "container arguments" to be sent off to the scheduler.
107105
"""
108106
if self.build is not None:
109107
if self.build.dryccfile:
110-
return self.build.dryccfile['deploy'].get(ptype, {}).get('args', [])
108+
return self.build.get_pipeline(ptype).get('deploy', {}).get('args', [])
111109
else:
112110
# dockerfile or container image
113111
if self.build.dockerfile or not self.build.sha:
@@ -119,11 +117,9 @@ def get_deploy_args(self, ptype):
119117

120118
def get_deploy_command(self, ptype):
121119
"""
122-
In the deploy phase of dryccfile
123120
Return the kubernetes "container command" to be sent off to the scheduler.
124121
"""
125-
return self.build.dryccfile.get(
126-
'deploy', {}).get(ptype, {}).get('command', [])
122+
return self.build.get_pipeline(ptype).get('deploy', {}).get('command', [])
127123

128124
def log(self, message, level=logging.INFO):
129125
"""Logs a message in the context of this application.

‎rootfs/api/serializers/__init__.py‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
from .schemas.volumes import SCHEMA as VOLUMES_SCHEMA
2525
from .schemas.autoscale import SCHEMA as AUTOSCALE_SCHEMA
2626
from .schemas.healthcheck import SCHEMA as HEALTHCHECK_SCHEMA
27-
from .schemas.dryccfile import (SCHEMA as DRYCCFILE_SCHEMA, PROCTYPE_REGEX)
27+
from .schemas.dryccfile import (SCHEMA as DRYCCFILE_SCHEMA, PROCTYPE_REGEX, CONFIGKEY_REGEX)
2828

2929

3030
User = get_user_model()
@@ -42,7 +42,7 @@
4242
PROCTYPE_MISMATCH_MSG = "Process types can only supports: %s" % PROCTYPE_MATCH.pattern
4343

4444
TAGVAL_MATCH = re.compile(r'^(?:[a-zA-Z\d][-\.\w]{0,61})?[a-zA-Z\d]$')
45-
CONFIGKEY_MATCH = re.compile(r'^[A-z0-9_\-\.]+$', re.IGNORECASE)
45+
CONFIGKEY_MATCH = re.compile(CONFIGKEY_REGEX, re.IGNORECASE)
4646
CONFIGKEY_MISMATCH_MSG = ("Config key can only supports: %s" % CONFIGKEY_MATCH.pattern)
4747
CONFIG_LIMITS_MISMATCH_MSG = "The limit plan {} does not exist"
4848

@@ -231,12 +231,15 @@ def validate_procfile(data):
231231

232232
@staticmethod
233233
def validate_dryccfile(data):
234+
ptypes = set()
234235
if data:
235236
validate_json(data, DRYCCFILE_SCHEMA, serializers.ValidationError)
236-
ptypes = set().union(data.get("deploy", {}).keys())
237-
ptypes = ptypes.union(data.get("build", {}).get("docker", {}).keys())
238-
for ptype in ptypes:
239-
validate_ptype(ptype)
237+
for pipeline in data['pipeline'].values():
238+
if pipeline['ptype'] in ptypes:
239+
raise serializers.ValidationError("Pipeline's ptype must be unique")
240+
else:
241+
validate_ptype(pipeline['ptype'])
242+
ptypes.add(pipeline['ptype'])
240243
return data
241244

242245

‎rootfs/api/serializers/schemas/dryccfile.py‎

Lines changed: 61 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1,121 +1,85 @@
1+
2+
FILENAME_REGEX = r'^(?P<name>[a-z0-9]+(\.[a-z0-9]+)*)$'
13
PROCTYPE_REGEX = r'^(?P<type>[a-z0-9]+(\-[a-z0-9]+)*)$'
4+
CONFIGKEY_REGEX = r'^[A-z0-9_\-\.]+$'
5+
CONFIGENV_SCHEMA = {
6+
"type": "object",
7+
"patternProperties": {
8+
CONFIGKEY_REGEX: {"type": "string"},
9+
},
10+
}
211
SCHEMA = {
312
"$schema": "http://json-schema.org/schema#",
413
"type": "object",
514
"properties": {
6-
"build": {
15+
"config": {
16+
"type": "object",
17+
"patternProperties": {
18+
FILENAME_REGEX: CONFIGENV_SCHEMA,
19+
},
20+
},
21+
"pipeline": {
722
"type": "object",
8-
"properties": {
9-
"docker": {
23+
"patternProperties": {
24+
FILENAME_REGEX: {
1025
"type": "object",
11-
"patternProperties": {
12-
PROCTYPE_REGEX: {"type": "string"},
13-
},
14-
"additionalProperties": False,
15-
},
16-
"config": {
17-
"oneOf": [
18-
{
26+
"properties": {
27+
"kind": {"type": "string", "pattern": "^pipeline$"},
28+
"ptype": {"type": "string", "pattern": PROCTYPE_REGEX},
29+
"build": {
1930
"type": "object",
20-
"patternProperties": {
21-
PROCTYPE_REGEX: {
31+
"properties": {
32+
"docker": {"type": "string"},
33+
"arg": CONFIGENV_SCHEMA,
34+
},
35+
"additionalProperties": False,
36+
},
37+
"env": CONFIGENV_SCHEMA,
38+
"run": {
39+
"type": "object",
40+
"properties": {
41+
"command": {
2242
"type": "array",
23-
"items": {
24-
"type": "object",
25-
"properties": {
26-
"name": {"type": "string"},
27-
"value": {"type": "string"},
28-
},
29-
"additionalProperties": False,
30-
},
43+
"items": {"type": "string"},
3144
},
45+
"args": {
46+
"type": "array",
47+
"items": {"type": "string"},
48+
},
49+
"image": {"type": "string"},
50+
"timeout": {"type": "integer"},
3251
},
33-
"minProperties": 1,
3452
"additionalProperties": False,
3553
},
36-
{
54+
"config": {
3755
"type": "array",
3856
"items": {
39-
"type": "object",
40-
"properties": {
41-
"name": {"type": "string"},
42-
"value": {"type": "string"},
43-
},
44-
"additionalProperties": False,
57+
"type": "string",
58+
"pattern": FILENAME_REGEX,
4559
},
4660
},
47-
]
48-
}
49-
},
50-
},
51-
"config": {
52-
"type": "array",
53-
"items": {
54-
"type": "object",
55-
"properties": {
56-
"name": {"type": "string"},
57-
"group": {"type": "string"},
58-
"value": {"type": "string"},
59-
},
60-
"additionalProperties": False,
61-
},
62-
},
63-
"run": {
64-
"type": "object",
65-
"patternProperties": {
66-
PROCTYPE_REGEX: {
67-
"properties": {
68-
"image": {"type": "string"},
69-
"command": {
70-
"type": "array",
71-
"items": {"type": "string"},
72-
},
73-
"args": {
74-
"type": "array",
75-
"items": {"type": "string"},
76-
}
77-
}
78-
},
79-
},
80-
"additionalProperties": False,
81-
},
82-
"deploy": {
83-
"type": "object",
84-
"patternProperties": {
85-
PROCTYPE_REGEX: {
86-
"properties": {
87-
"image": {"type": "string"},
88-
"command": {
89-
"type": "array",
90-
"items": {"type": "string"},
91-
},
92-
"args": {
93-
"type": "array",
94-
"items": {"type": "string"},
95-
},
96-
"config": {
97-
"env": {
98-
"type": "array",
99-
"items": {
100-
"type": "object",
101-
"properties": {
102-
"name": {"type": "string"},
103-
"value": {"type": "string"},
104-
},
105-
"additionalProperties": False,
61+
"deploy": {
62+
"type": "object",
63+
"properties": {
64+
"command": {
65+
"type": "array",
66+
"items": {"type": "string"},
10667
},
68+
"args": {
69+
"type": "array",
70+
"items": {"type": "string"},
71+
},
72+
"image": {"type": "string"},
10773
},
108-
"ref": {
109-
"type": "array",
110-
"items": {"type": "string"},
111-
}
112-
},
113-
}
74+
"additionalProperties": False,
75+
}
76+
},
77+
"additionalProperties": False,
78+
"required": ["kind", "ptype", "deploy"],
11479
},
11580
},
116-
"minProperties": 1,
117-
"additionalProperties": False,
11881
},
11982
},
120-
"required": ["deploy"],
83+
"required": ["pipeline"],
84+
"additionalProperties": False,
12185
}

0 commit comments

Comments
 (0)