From af391977c2043fe1a57f9d4c7a669ea207b5716a Mon Sep 17 00:00:00 2001
From: Your Name <you@example.com>
Date: Sat, 29 Aug 2020 07:57:45 +0800
Subject: [PATCH] feat(controller): add a volume command

---
 .../templates/controller-clusterrole.yaml     |   3 +
 .../templates/controller-deployment.yaml      |   4 +
 charts/controller/values.yaml                 |   3 +
 rootfs/api/migrations/0001_initial.py         |  18 ++
 rootfs/api/models/__init__.py                 |   2 +
 rootfs/api/models/app.py                      |  35 +++-
 rootfs/api/models/volume.py                   |  73 ++++++++
 rootfs/api/serializers.py                     |  41 ++++-
 rootfs/api/settings/production.py             |   2 +
 rootfs/api/settings/testing.py                |   2 +
 rootfs/api/tests/test_volume.py               | 163 ++++++++++++++++++
 rootfs/api/urls.py                            |   7 +
 rootfs/api/views.py                           |  99 ++++++++++-
 rootfs/scheduler/mock.py                      |   4 +-
 rootfs/scheduler/resources/deployment.py      |  13 +-
 rootfs/scheduler/resources/pod.py             |  13 ++
 rootfs/scheduler/resources/pvc.py             |  76 ++++++++
 rootfs/scheduler/tests/test_pvc.py            |  79 +++++++++
 18 files changed, 625 insertions(+), 12 deletions(-)
 create mode 100644 rootfs/api/models/volume.py
 create mode 100644 rootfs/api/tests/test_volume.py
 create mode 100644 rootfs/scheduler/resources/pvc.py
 create mode 100644 rootfs/scheduler/tests/test_pvc.py

diff --git a/charts/controller/templates/controller-clusterrole.yaml b/charts/controller/templates/controller-clusterrole.yaml
index d2b120450..4792a2c0c 100644
--- a/charts/controller/templates/controller-clusterrole.yaml
+++ b/charts/controller/templates/controller-clusterrole.yaml
@@ -62,5 +62,8 @@ rules:
 - apiGroups: ["apps"]
   resources: ["replicasets"]
   verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["persistentvolumeclaims"]
+  verbs: ["get", "list", "watch", "create", "delete", "patch", "update"]
 {{- end -}}
 {{- end -}}
diff --git a/charts/controller/templates/controller-deployment.yaml b/charts/controller/templates/controller-deployment.yaml
index b40c79dd3..9476df4e6 100644
--- a/charts/controller/templates/controller-deployment.yaml
+++ b/charts/controller/templates/controller-deployment.yaml
@@ -75,6 +75,10 @@ spec:
               value: "{{ .Values.global.registry_secret_prefix }}"
             - name: "IMAGE_PULL_POLICY"
               value: "{{ .Values.app_pull_policy }}"
+{{- if (.Values.app_storage_class) }}
+            - name: "DRYCC_APP_KUBERNETES_STORAGE_CLASS"
+              value: "{{ .Values.app_storage_class }}"
+{{- end }}
             - name: "TZ"
               value: {{ .Values.time_zone | default "UTC" | quote }}
 {{- if (.Values.deploy_hook_urls) }}
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index 06eea18b9..65d0847f1 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -16,6 +16,9 @@ registration_mode: "admin_only"
 # Option to disable ssl verification to connect to k8s api server
 k8s_api_verify_tls: "true"
 
+# Set storageClassName, It is used for application mount.
+app_storage_class: ""
+
 global:
   # Admin email, used for each component to send email to administrator
   email: "drycc@drycc.cc"
diff --git a/rootfs/api/migrations/0001_initial.py b/rootfs/api/migrations/0001_initial.py
index da499da40..c8a39c2f2 100644
--- a/rootfs/api/migrations/0001_initial.py
+++ b/rootfs/api/migrations/0001_initial.py
@@ -233,4 +233,22 @@ class Migration(migrations.Migration):
             name='appsettings',
             unique_together=set([('app', 'uuid')]),
         ),
+        migrations.CreateModel(
+            name='Volume',
+            fields=[
+                ('uuid', models.UUIDField(auto_created=True, default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='UUID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('name', models.CharField(max_length=63, unique=True, validators=[api.models.validate_label])),
+                ('size', models.CharField(max_length=128)),
+                ('path', jsonfield.fields.JSONField(blank=True, default={})),
+                ('app', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.App')),
+                ('owner', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'ordering': ['-created'],
+                'get_latest_by': 'created',
+                'unique_together': {('app', 'name')},
+            },
+        ),
     ]
diff --git a/rootfs/api/models/__init__.py b/rootfs/api/models/__init__.py
index 94aba8dc3..499e93757 100644
--- a/rootfs/api/models/__init__.py
+++ b/rootfs/api/models/__init__.py
@@ -150,6 +150,7 @@ class Meta:
 from .key import Key, validate_base64  # noqa
 from .release import Release  # noqa
 from .tls import TLS  # noqa
+from .volume import Volume  # noqa
 
 # define update/delete callbacks for synchronizing
 # models with the configuration management backend
@@ -246,6 +247,7 @@ def _hook_release_created(**kwargs):
 post_delete.connect(_log_instance_removed, sender=Certificate, dispatch_uid='api.models.log')
 post_delete.connect(_log_instance_removed, sender=Domain, dispatch_uid='api.models.log')
 post_delete.connect(_log_instance_removed, sender=TLS, dispatch_uid='api.models.log')
+post_delete.connect(_log_instance_removed, sender=Volume, dispatch_uid='api.models.log')
 
 
 # automatically generate a new token on creation
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 34f3d8d8e..688f2013d 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -11,6 +11,7 @@
 import requests
 import string
 import time
+import uuid
 from urllib.parse import urljoin
 
 from django.conf import settings
@@ -25,6 +26,7 @@
 from api.models.release import Release
 from api.models.tls import TLS
 from api.models.appsettings import AppSettings
+from api.models.volume import Volume
 from api.utils import generate_app_name, async_run
 from scheduler import KubeHTTPException, KubeException
 
@@ -497,7 +499,7 @@ def scale(self, user, structure):  # noqa
     def _scale_pods(self, scale_types):
         release = self.release_set.filter(failed=False).latest()
         app_settings = self.appsettings_set.latest()
-
+        volumes = Volume.objects.filter(app=self, path__isnull=False)
         # use slugrunner image for app if buildpack app otherwise use normal image
         if release.build.type == 'buildpack':
             image = next(filter(lambda item: item['name'] == release.build.stack,
@@ -507,7 +509,8 @@ def _scale_pods(self, scale_types):
 
         tasks = []
         for scale_type, replicas in scale_types.items():
-            data = self._gather_app_settings(release, app_settings, scale_type, replicas)  # noqa
+            scale_type_volumes = [_ for _ in volumes if scale_type in _.path.keys()]
+            data = self._gather_app_settings(release, app_settings, scale_type, replicas, volumes=scale_type_volumes)  # noqa
 
             # gather all proc types to be deployed
             tasks.append(
@@ -577,9 +580,11 @@ def deploy(self, release, force_deploy=False, rollback_on_failure=True):  # noqa
 
         # deploy application to k8s. Also handles initial scaling
         app_settings = self.appsettings_set.latest()
+        volumes = self.volume_set.all()
         deploys = {}
         for scale_type, replicas in self.structure.items():
-            deploys[scale_type] = self._gather_app_settings(release, app_settings, scale_type, replicas)  # noqa
+            volumes = [_ for _ in volumes if scale_type in _.path.keys()]
+            deploys[scale_type] = self._gather_app_settings(release, app_settings, scale_type, replicas, volumes=volumes)  # noqa
 
         # Sort deploys so routable comes first
         deploys = OrderedDict(sorted(deploys.items(), key=lambda d: d[1].get('routable')))
@@ -798,7 +803,7 @@ def logs(self, log_lines=str(settings.LOG_LINES)):
         # cast content to string since it comes as bytes via the requests object
         return str(r.content.decode('utf-8'))
 
-    def run(self, user, command):
+    def run(self, user, command, volumes=None):
         def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
             return ''.join(random.choice(chars) for _ in range(size))
 
@@ -814,8 +819,13 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
                                 settings.SLUGRUNNER_IMAGES))['image']
         else:
             image = release.image
-
-        data = self._gather_app_settings(release, app_settings, process_type='run', replicas=1)
+        volume_list = []
+        if volumes:
+            volume_objs = Volume.objects.filter(app=release.app, name__in=volumes.keys())
+            for _ in volume_objs:
+                _.path["{}-{}".format(self.app.id, str(uuid.uuid4())[:7])] = volumes.get(_.name, None)  # noqa
+                volume_list.append(_)
+        data = self._gather_app_settings(release, app_settings, process_type='run', replicas=1, volumes=volume_list)  # noqa
 
         # create application config and build the pod manifest
         self.set_application_config(release)
@@ -1076,7 +1086,7 @@ def _get_private_registry_config(self, image, registry=None):
         })
         return docker_config, name, True
 
-    def _gather_app_settings(self, release, app_settings, process_type, replicas):
+    def _gather_app_settings(self, release, app_settings, process_type, replicas, volumes=None):
         """
         Gathers all required information needed in one easy place for passing into
         the Kubernetes client to deploy an application
@@ -1116,6 +1126,15 @@ def _gather_app_settings(self, release, app_settings, process_type, replicas):
         healthcheck = config.get_healthcheck().get(process_type, {})
         if not healthcheck and process_type in ['web', 'cmd']:
             healthcheck = config.get_healthcheck().get('web/cmd', {})
+        volumes_info = [{
+            "name": _.name,
+            "claimName": _.name,
+        } for _ in volumes] if volumes else []
+
+        volume_mounts_info = [{
+            "name": _.name,
+            "mount_path": _.path.get(process_type),
+        } for _ in volumes] if volumes else []
 
         return {
             'memory': config.memory,
@@ -1140,6 +1159,8 @@ def _gather_app_settings(self, release, app_settings, process_type, replicas):
             'pod_termination_grace_period_each': config.termination_grace_period,
             'image_pull_secret_name': image_pull_secret_name,
             'image_pull_policy': image_pull_policy,
+            'volumes': volumes_info,
+            'volume_mounts': volume_mounts_info,
         }
 
     def set_application_config(self, release):
diff --git a/rootfs/api/models/volume.py b/rootfs/api/models/volume.py
new file mode 100644
index 000000000..d90568b71
--- /dev/null
+++ b/rootfs/api/models/volume.py
@@ -0,0 +1,73 @@
+import logging
+from django.db import models, transaction
+from django.conf import settings
+from jsonfield import JSONField
+from api.exceptions import DryccException, ServiceUnavailable, AlreadyExists
+from api.models import UuidAuditedModel, validate_label
+from scheduler import KubeException
+
+logger = logging.getLogger(__name__)
+
+
+class Volume(UuidAuditedModel):
+    owner = models.ForeignKey(settings.AUTH_USER_MODEL,
+                              on_delete=models.PROTECT)
+    app = models.ForeignKey('App', on_delete=models.CASCADE)
+
+    name = models.CharField(max_length=63, unique=True,
+                            validators=[validate_label])
+    size = models.CharField(max_length=128, blank=False, null=False,
+                            unique=False)
+    path = JSONField(default={}, blank=True)
+
+    class Meta:
+        get_latest_by = 'created'
+        unique_together = (('app', 'name'),)
+        ordering = ['-created']
+
+    def __str__(self):
+        return self.name
+
+    @transaction.atomic
+    def save(self, *args, **kwargs):
+        # Attach volume, updates k8s
+        if self.created == self.updated:
+            self.attach(*args, **kwargs)
+        # Save to DB
+        return super(Volume, self).save(*args, **kwargs)
+
+    @transaction.atomic
+    def delete(self, *args, **kwargs):
+        if self.path:
+            raise DryccException("the volume is not unmounted")
+        # Deatch volume, updates k8s
+        self.detach(*args, **kwargs)
+        # Delete from DB
+        return super(Volume, self).delete(*args, **kwargs)
+
+    def attach(self, *args, **kwargs):
+        try:
+            self._scheduler.pvc.get(self.app.id, self.name)
+            err = "Volume {} already exists in this namespace".format(self.name)  # noqa
+            self.log(err, logging.INFO)
+            raise AlreadyExists(err)
+        except KubeException as e:
+            logger.info(e)
+            try:
+                kwargs = {
+                    "size": self.size
+                }
+                self._scheduler.pvc.create(self.app.id, self.name, **kwargs)
+            except KubeException as e:
+                msg = 'There was a problem creating the volume ' \
+                      '{} for {}'.format(self.name, self.app_id)
+                raise ServiceUnavailable(msg) from e
+
+    def detach(self, *args, **kwargs):
+        try:
+            # We raise an exception when a volume doesn't exist
+            self._scheduler.pvc.get(self.app.id, self.name)
+            self._scheduler.pvc.delete(self.app.id, self.name)
+        except KubeException as e:
+            raise ServiceUnavailable("Could not delete volume {} for application \
+                {}".format(name, self.app_id)) from e  # noqa
diff --git a/rootfs/api/serializers.py b/rootfs/api/serializers.py
index 88ac03b37..e103dde1d 100644
--- a/rootfs/api/serializers.py
+++ b/rootfs/api/serializers.py
@@ -28,6 +28,10 @@
 TAGVAL_MATCH = re.compile(r'^(?:[a-zA-Z\d][-\.\w]{0,61})?[a-zA-Z\d]$')
 CONFIGKEY_MATCH = re.compile(r'^[a-z_]+[a-z0-9_]*$', re.IGNORECASE)
 TERMINATION_GRACE_PERIOD_MATCH = re.compile(r'^[0-9]*$')
+VOLUME_SIZE_MATCH = re.compile(
+    r'^(?P<mem>(([0-9]+(MB|KB|GB|[BKMG])|0)(/([0-9]+(MB|KB|GB|[BKMG])))?))$', re.IGNORECASE)
+VOLUME_PATH = re.compile(r'^\/(\w+\/?)+$', re.IGNORECASE)
+
 PROBE_SCHEMA = {
     "$schema": "http://json-schema.org/schema#",
 
@@ -607,6 +611,41 @@ class TLSSerializer(serializers.ModelSerializer):
     owner = serializers.ReadOnlyField(source='owner.username')
 
     class Meta:
-        """Metadata options for a :class:`AppSettingsSerializer`."""
+        """Metadata options for a :class:`AppTLSSerializer`."""
         model = models.TLS
         fields = '__all__'
+
+
+class VolumeSerializer(serializers.ModelSerializer):
+    """Serialize a :class:`~api.models.Volume` model."""
+
+    app = serializers.SlugRelatedField(slug_field='id', queryset=models.App.objects.all())
+    owner = serializers.ReadOnlyField(source='owner.username')
+    name = serializers.CharField()
+    size = serializers.CharField()
+    path = JSONFieldSerializer(required=False, binary=True)
+
+    class Meta:
+        """Metadata options for a :class:`AppVolumeSerializer`."""
+        model = models.Volume
+        fields = '__all__'
+
+    def validate_size(self, data):
+        if not re.match(VOLUME_SIZE_MATCH, str(data)):
+            raise serializers.ValidationError(
+                "Volume size limit format: <number><unit> or <number><unit>/<number><unit>, "
+                "where unit = B, K, M or G")
+        return data
+
+    def validate_path(self, data):
+        for key, value in data.items():
+            if value is None:  # use NoneType to unset an item
+                continue
+
+            if not re.match(PROCTYPE_MATCH, key):
+                raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
+
+            if not re.match(VOLUME_PATH, str(value)):
+                raise serializers.ValidationError(
+                    "Volume path format: /path")
+        return data
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index f57e008c9..b15db544e 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -326,6 +326,8 @@
 
 DRYCC_DEPLOY_HOOK_SECRET_KEY = os.environ.get('DRYCC_DEPLOY_HOOK_SECRET_KEY', None)
 
+DRYCC_APP_KUBERNETES_STORAGE_CLASS = os.environ.get('DRYCC_APP_KUBERNETES_STORAGE_CLASS', "")  # noqa
+
 KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT = os.environ.get('KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT', None)  # noqa
 
 DRYCC_DEFAULT_CONFIG_TAGS = os.environ.get('DRYCC_DEFAULT_CONFIG_TAGS', '')
diff --git a/rootfs/api/settings/testing.py b/rootfs/api/settings/testing.py
index 96a4841df..782cc1a0b 100644
--- a/rootfs/api/settings/testing.py
+++ b/rootfs/api/settings/testing.py
@@ -42,6 +42,8 @@
 
 DRYCC_DEFAULT_CONFIG_TAGS = os.environ.get('DRYCC_DEFAULT_CONFIG_TAGS', '')
 
+DRYCC_APP_KUBERNETES_STORAGE_CLASS = os.environ.get('DRYCC_APP_KUBERNETES_STORAGE_CLASS', '')  # noqa
+
 
 class DisableMigrations(object):
 
diff --git a/rootfs/api/tests/test_volume.py b/rootfs/api/tests/test_volume.py
new file mode 100644
index 000000000..2238e223c
--- /dev/null
+++ b/rootfs/api/tests/test_volume.py
@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+"""
+Unit tests for the Drycc api app.
+
+Run the tests with "./manage.py test api"
+"""
+from django.contrib.auth.models import User
+from django.core.cache import cache
+from django.conf import settings
+from rest_framework.authtoken.models import Token
+from api.tests import adapter, DryccTransactionTestCase
+import requests_mock
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class VolumeTest(DryccTransactionTestCase):
+    """Tests setting and updating config values"""
+
+    fixtures = ['tests.json']
+
+    def setUp(self):
+        self.user = User.objects.get(username='autotest')
+        self.token = Token.objects.get(user=self.user).key
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        self.app_id = self.create_app()
+
+    def tearDown(self):
+        # Restore default tags to empty string
+        settings.DRYCC_DEFAULT_CONFIG_TAGS = ''
+        # make sure every test has a clean slate for k8s mocking
+        cache.clear()
+
+    def test_volumecreate(self, mock_requests):
+        """Test that the serialized response contains only relevant data."""
+        app_id = self.create_app()
+
+        response = self.client.post(
+            '/v2/apps/{}/volumes'.format(app_id),
+            data={'name': 'myvolume', 'size': '500M'}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+
+        for key in response.data:
+            self.assertIn(key,
+                          ['uuid', 'owner', 'created', 'updated', 'app', 'name',
+                           'size', 'path'])
+
+        expected = {
+            'owner': self.user.username,
+            'app': app_id,
+            'name': 'myvolume',
+            'size': '500M'
+        }
+        self.assertDictContainsSubset(expected, response.data)
+
+    def test_volume_list_unmount(self, mock_requests):
+        """
+        Test that volume is auto-created for a new app and that
+        volume can be updated using a PATCH
+        """
+        # create
+        app_id = self.create_app()
+        data = [
+            {'name': 'myvolume1', 'size': '500M'},
+            {'name': 'myvolume2', 'size': '500M'}
+        ]
+        for _ in data:
+            self.client.post('/v2/apps/{}/volumes'.format(app_id), data=_)
+        # Fetch
+        url = '/v2/apps/{app_id}/volumes'.format(app_id=app_id)
+        response = self.client.get(url)
+        expected = [res['name'] for res in response.data['results']]
+        self.assertEqual(sorted([_['name'] for _ in data]), sorted(expected))
+
+    def test_volume_delete(self, mock_requests):
+        """
+        Test that volume is delete for a new app and that
+        volume can be updated using a PATCH
+        """
+        # create
+        app_id = self.create_app()
+        data = [
+            {'name': 'myvolume1', 'size': '500M'},
+            {'name': 'myvolume2', 'size': '500M'}
+        ]
+        for _ in data:
+            self.client.post('/v2/apps/{}/volumes'.format(app_id), data=_)
+
+        # Delete
+        url = '/v2/apps/{app_id}/volumes/{volume}'.format(app_id=app_id,
+                                                          volume='myvolume1')
+        response = self.client.delete(url)
+        self.assertEqual(response.status_code, 204)
+
+        # Fetch
+        url = '/v2/apps/{app_id}/volumes'.format(app_id=app_id)
+        response = self.client.get(url)
+        expected = [res['name'] for res in response.data['results']]
+        self.assertEqual(
+            sorted([_['name'] for _ in data if _['name'] != 'myvolume1']),
+            sorted(expected))  # noqa
+
+    def test_volume_mount(self, mock_requests):
+        # create
+        app_id = self.create_app()
+        data = [
+            {'name': 'myvolume1', 'size': '500M'}
+        ]
+        for _ in data:
+            self.client.post('/v2/apps/{}/volumes'.format(app_id), data=_)
+
+        self.build_deploy(app_id)
+        # mount
+        url = '/v2/apps/{app_id}/volumes/myvolume1/path'.format(app_id=app_id)
+        mount_path = {"path": {"web": "/data/web1"}}
+        response = self.client.patch(url, data=mount_path)
+        expected = response.data['path']
+        self.assertEqual(mount_path['path'], expected)
+
+    def test_volume_unmount(self, mock_requests):
+        # create
+        app_id = self.create_app()
+        data = [
+            {'name': 'myvolume1', 'size': '500M'}
+        ]
+        for _ in data:
+            self.client.post('/v2/apps/{}/volumes'.format(app_id), data=_)
+
+        # Fetch
+        url = '/v2/apps/{app_id}/volumes'.format(app_id=app_id)
+        response = self.client.get(url)
+        expected = [res['path'] for res in response.data['results']]
+        self.assertEqual({}, expected[0])
+
+        self.build_deploy(app_id)
+        # mount
+        url = '/v2/apps/{app_id}/volumes/myvolume1/path'.format(app_id=app_id)
+        mount_path = {"path": {"web": "/data/web1"}}
+        response = self.client.patch(url, data=mount_path)
+        expected = response.data['path']
+        self.assertEqual(mount_path['path'], expected)
+
+        # unmount
+        url = '/v2/apps/{app_id}/volumes/myvolume1/path'.format(app_id=app_id)
+        mount_path = {"path": {"web": None}}
+        response = self.client.patch(url, data=mount_path)
+        expected = response.data['path']
+        self.assertEqual({}, expected)
+
+    def build_deploy(self, app_id):
+        # post a new build with procfile
+        url = "/v2/apps/{app_id}/builds".format(app_id=app_id)
+        body = {
+            'image': 'autotest/example',
+            'sha': 'a'*40,
+            'stack': 'heroku-18',
+            'procfile': {
+                'web': 'node server.js',
+                'worker': 'node worker.js'
+            }
+        }
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index d1adfcc98..936e214f1 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -75,6 +75,13 @@
         views.AppPermsViewSet.as_view({'delete': 'destroy'})),
     url(r"^apps/(?P<id>{})/perms/?$".format(settings.APP_URL_REGEX),
         views.AppPermsViewSet.as_view({'get': 'list', 'post': 'create'})),
+    # application volumes
+    url(r"^apps/(?P<id>{})/volumes/?$".format(settings.APP_URL_REGEX),
+        views.AppVolumesViewSet.as_view({'get': 'list', 'post': 'create'})),
+    url(r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/?$".format(settings.APP_URL_REGEX),
+        views.AppVolumesViewSet.as_view({'delete': 'destroy'})),
+    url(r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/path/?$".format(settings.APP_URL_REGEX),
+        views.AppVolumeMountPathViewSet.as_view({'patch': 'path'})),
     # apps base endpoint
     url(r"^apps/(?P<id>{})/?$".format(settings.APP_URL_REGEX),
         views.AppViewSet.as_view({'get': 'retrieve', 'post': 'update', 'delete': 'destroy'})),
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index b4cbbdf69..9fb9e7d71 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -1,6 +1,8 @@
 """
 RESTful view classes for presenting Drycc API objects.
 """
+from copy import deepcopy
+
 from django.http import Http404, HttpResponse
 from django.conf import settings
 from django.contrib.auth.models import User
@@ -17,10 +19,13 @@
 from rest_framework.authtoken.models import Token
 
 from api import authentication, models, permissions, serializers, viewsets
-from api.models import AlreadyExists, ServiceUnavailable, DryccException, UnprocessableEntity
+from api.models import AlreadyExists, ServiceUnavailable, DryccException, \
+    UnprocessableEntity
 
 import logging
 
+from api.serializers import VolumeSerializer
+
 logger = logging.getLogger(__name__)
 
 
@@ -238,7 +243,10 @@ def run(self, request, **kwargs):
         app = self.get_object()
         if not request.data.get('command'):
             raise DryccException("command is a required field")
-        rc, output = app.run(self.request.user, request.data['command'])
+        volumes = request.data.get('volumes', None)
+        if volumes:
+            VolumeSerializer().validate_path(volumes)
+        rc, output = app.run(self.request.user, request.data['command'], volumes)
         return Response({'exit_code': rc, 'output': str(output)})
 
     def update(self, request, **kwargs):
@@ -630,6 +638,93 @@ def destroy(self, request, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
+class AppVolumesViewSet(ReleasableViewSet):
+    """RESTful views for volumes apps with collaborators."""
+    model = models.Volume
+    serializer_class = serializers.VolumeSerializer
+
+    def destroy(self, request, **kwargs):
+        volume = get_object_or_404(models.Volume,
+                                   app__id=self.kwargs['id'],
+                                   name=self.kwargs['name'])
+        volume.delete()
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
+
+class AppVolumeMountPathViewSet(ReleasableViewSet):
+    serializer_class = serializers.VolumeSerializer
+
+    def get_object(self):
+        return get_object_or_404(models.Volume,
+                                 app__id=self.kwargs['id'],
+                                 name=self.kwargs['name'])
+
+    def path(self, request, *args, **kwargs):
+        new_path = request.data.get('path')
+        if new_path is None:
+            raise DryccException("path is a required field")
+        obj = self.get_object()
+        container_types = [_ for _ in new_path.keys()
+                           if _ not in obj.app.procfile_structure.keys()]
+        if container_types:
+            raise DryccException("process type {} is not included in profile".
+                                 format(','.join(container_types)))
+        path = obj.path
+        pre_path = deepcopy(path)
+        # merge mount path
+        # remove path keys if a null value is provided
+        for key, value in new_path.items():
+            if value is None:
+                # error if unsetting non-existing key
+                if key not in path:
+                    raise UnprocessableEntity(
+                        '{} does not exist under {}'.format(key, pre_path.keys()))  # noqa
+                path.pop(key)
+            else:
+                path[key] = value
+        obj.path = path  # after merge path
+        obj.save()
+        self.deploy(obj, pre_path)
+        serializer = self.get_serializer(obj, many=False)
+        return Response(serializer.data)
+
+    def deploy(self, volume, pre_mount_path):
+        app = self.get_app()
+        latest_release = app.release_set.filter(failed=False).latest()
+        latest_version = app.release_set.latest().version
+        try:
+            summary = "{user} deployed {app},".\
+                format(user=self.request.user, app=app.id)
+            for container_type, path in volume.path.items():
+                summary += "{container_type} mount path {path} with volume {name}.".\
+                               format(container_type=container_type, path=path, name=volume.name)  # noqa
+            print(summary)
+            self.release = latest_release.new(
+                self.request.user,
+                config=latest_release.config,
+                build=latest_release.build,
+                summary=summary)
+            # It's possible to mount volume before a build
+            if self.release.build is not None:
+                app.deploy(self.release)
+        except Exception as e:
+            if (not hasattr(self, 'release') and
+                    app.release_set.latest().version == latest_version+1):
+                self.release = app.release_set.latest()
+            if hasattr(self, 'release'):
+                self.release.failed = True
+                self.release.summary = "{} deploy with a volume that failed".\
+                    format(self.request.user)  # noqa
+                # Get the exception that has occured
+                self.release.exception = "error: {}".format(str(e))
+                self.release.save()
+            volume.path = pre_mount_path
+            volume.save()
+            if isinstance(e, AlreadyExists):
+                raise
+            raise DryccException(str(e)) from e
+
+
 class AdminPermsViewSet(BaseDryccViewSet):
     """RESTful views for sharing admin permissions with other users."""
 
diff --git a/rootfs/scheduler/mock.py b/rootfs/scheduler/mock.py
index cd4bb74ca..d734116ee 100644
--- a/rootfs/scheduler/mock.py
+++ b/rootfs/scheduler/mock.py
@@ -80,7 +80,7 @@ def _acquire(self):
 resources = [
     'namespaces', 'nodes', 'pods', 'replicationcontrollers',
     'secrets', 'services', 'events', 'deployments', 'replicasets',
-    'horizontalpodautoscalers', 'scale', 'resourcequotas', 'ingresses'
+    'horizontalpodautoscalers', 'scale', 'resourcequotas', 'ingresses', 'persistentvolumeclaims'
 ]
 
 
@@ -1063,3 +1063,5 @@ def __init__(self, url, k8s_api_verify_tls=True):
 # GET                                       /namespaces/{namespace}/pods/{pod}/log  (needs to be special cased)  # noqa
 # POST (NI) | GET                           /namespaces/{namespace}/deployments  # noqa
 # PATCH (NI) | PUT (NI) | GET | DELETE      /namespaces/{namespace}/deployments/{deployment}  # noqa
+# POST (NI) | GET                           /namespaces/{namespace}/persistentvolumeclaims  # noqa
+# DELETE(NI) | GET                          /namespaces/{namespace}/persistentvolumeclaims/{persistentvolumeclaim}  # noqa
diff --git a/rootfs/scheduler/resources/deployment.py b/rootfs/scheduler/resources/deployment.py
index d3fcbe8ac..215ed60c4 100644
--- a/rootfs/scheduler/resources/deployment.py
+++ b/rootfs/scheduler/resources/deployment.py
@@ -34,7 +34,7 @@ def manifest(self, namespace, name, image, entrypoint, command, spec_annotations
         replicas = kwargs.get('replicas', 0)
         batches = kwargs.get('deploy_batches', None)
         tags = kwargs.get('tags', {})
-
+        volumes = kwargs.get('volumes', [])
         labels = {
             'app': namespace,
             'type': kwargs.get('app_type'),
@@ -107,6 +107,17 @@ def manifest(self, namespace, name, image, entrypoint, command, spec_annotations
         # set the old deployment spec annotations on this deployment
         manifest['spec']['template']['metadata']['annotations'] = spec_annotations
 
+        # set volumes
+        if volumes:
+            exist_volumes = manifest['spec']['template']['spec'].get("volumes", [])  # noqa
+            for volume in volumes:
+                exist_volumes.append(
+                    {"name": volume.get('name'),
+                     "persistentVolumeClaim": {
+                         "claimName": volume.get('claimName')
+                     }})
+            manifest['spec']['template']['spec']['volumes'] = exist_volumes
+
         return manifest
 
     def create(self, namespace, name, image, entrypoint, command, spec_annotations, **kwargs):
diff --git a/rootfs/scheduler/resources/pod.py b/rootfs/scheduler/resources/pod.py
index c637b5f03..416f566be 100644
--- a/rootfs/scheduler/resources/pod.py
+++ b/rootfs/scheduler/resources/pod.py
@@ -225,6 +225,19 @@ def _set_container(self, namespace, container_name, data, **kwargs):
 
         self._set_lifecycle_hooks(data, env, **kwargs)
 
+        self._set_volume_mounts(data, kwargs)
+
+    @staticmethod
+    def _set_volume_mounts(container, kwargs):
+        # set volume_mounts
+        volume_mounts = kwargs.get('volume_mounts', [])
+        exist_volume_mounts = container.get('volumeMounts', [])
+        for volume_mount in volume_mounts:
+            exist_volume_mounts.append(
+                {"name": volume_mount.get('name'),
+                 "mountPath": volume_mount.get('mount_path')})
+        container['volumeMounts'] = exist_volume_mounts
+
     def _set_resources(self, container, kwargs):
         """ Set CPU/memory resource management manifest """
         app_type = kwargs.get("app_type")
diff --git a/rootfs/scheduler/resources/pvc.py b/rootfs/scheduler/resources/pvc.py
new file mode 100644
index 000000000..214f499bf
--- /dev/null
+++ b/rootfs/scheduler/resources/pvc.py
@@ -0,0 +1,76 @@
+from api.settings.production import DRYCC_APP_KUBERNETES_STORAGE_CLASS
+from scheduler.resources import Resource
+from scheduler.exceptions import KubeHTTPException
+
+
+class PVC(Resource):
+    @staticmethod
+    def manifest(namespace, name, version=None, **kwargs):
+        labels = {
+            'heritage': 'drycc',
+        }
+        data = {
+            "apiVersion": "v1",
+            "kind": "PersistentVolumeClaim",
+            "metadata": {
+                "name": name,
+                "namespace": namespace,
+                'labels': labels
+            },
+            "spec": {
+                "accessModes": [
+                    "ReadWriteMany"
+                ],
+                "resources": {
+                    "requests": {
+                        "storage": kwargs.get('size')
+                    },
+                },
+                "storageClassName": DRYCC_APP_KUBERNETES_STORAGE_CLASS,
+                "volumeMode": "Filesystem",
+            }
+        }
+        if version:
+            data["metadata"]["resourceVersion"] = version
+        return data
+
+    def get(self, namespace, name=None):
+        """
+        Fetch a single persistentvolumeclaim or a list of persistentvolumeclaim
+        """
+        if name is not None:
+            url = self.api('/namespaces/{}/persistentvolumeclaims/{}',
+                           namespace, name)
+            message = 'get persistentvolumeclaim ' + name
+        else:
+            url = self.api('/namespaces/{}/persistentvolumeclaims', namespace)
+            message = 'get persistentvolumeclaims'
+        response = self.http_get(url)
+        if self.unhealthy(response.status_code):
+            raise KubeHTTPException(response, message)
+        return response
+
+    def create(self, namespace, name, **kwargs):
+        """
+        Create persistentvolumeclaim
+        """
+        url = self.api('/namespaces/{}/persistentvolumeclaims', namespace)
+        data = self.manifest(namespace, name, **kwargs)
+        response = self.http_post(url, json=data)
+        if not response.status_code == 201:
+            raise KubeHTTPException(
+                response,
+                "create persistentvolumeclaim {}".format(namespace))
+        return response
+
+    def delete(self, namespace, name):
+        """
+        Delete persistentvolumeclaim
+        """
+        url = self.api('/namespaces/{}/persistentvolumeclaims/{}', namespace,
+                       name)
+        response = self.http_delete(url)
+        if self.unhealthy(response.status_code):
+            raise KubeHTTPException(response,
+                                    'delete persistentvolumeclaim ' + name)
+        return response
diff --git a/rootfs/scheduler/tests/test_pvc.py b/rootfs/scheduler/tests/test_pvc.py
new file mode 100644
index 000000000..dcafcbb33
--- /dev/null
+++ b/rootfs/scheduler/tests/test_pvc.py
@@ -0,0 +1,79 @@
+"""
+Unit tests for the Drycc scheduler module.
+
+Run the tests with './manage.py test scheduler'
+"""
+from scheduler import KubeHTTPException
+from scheduler.tests import TestCase
+from scheduler.utils import generate_random_name
+
+
+class PVCTest(TestCase):
+    """Tests scheduler pod calls"""
+
+    def create(self, namespace=None, name=generate_random_name(), **kwargs):
+        """
+        Helper function to create and verify a pvc on the namespace
+        """
+        namespace = self.namespace if namespace is None else namespace
+        # these are all required even if it is kwargs...
+        kwargs = {
+            'size': '500M'
+        }
+        pvc = self.scheduler.pvc.create(namespace, name, **kwargs)
+        self.assertEqual(pvc.status_code, 201, pvc.json())
+        return name
+
+    def test_create_failure(self):
+        with self.assertRaises(
+            KubeHTTPException,
+            msg='failed to create pvc doesnotexist in Namespace {}: 404 Not Found'.format(self.namespace)  # noqa
+        ):
+            self.create('doesnotexist', 'doesnotexist')
+
+    def test_create(self):
+        self.scheduler.ns.create("test-pvc")
+        self.create(namespace="test-pvc")
+
+    def test_delete_failure(self):
+        # test failure
+        with self.assertRaises(
+            KubeHTTPException,
+            msg='failed to delete pvc foo in Namespace {}: 404 Not Found'.format(self.namespace)  # noqa
+        ):
+            self.scheduler.pvc.delete(self.namespace, 'foo')
+
+    def test_delete(self):
+        # test success
+        name = self.create()
+        response = self.scheduler.pvc.delete(self.namespace, name)
+        data = response.json()
+        self.assertEqual(response.status_code, 200, data)
+
+    def test_get_pvcs(self):
+        # test success
+        name = self.create()
+        response = self.scheduler.pvc.get(self.namespace)
+        data = response.json()
+        self.assertEqual(response.status_code, 200, data)
+        self.assertIn('items', data)
+        self.assertEqual(1, len(data['items']), data['items'])
+        # simple verify of data
+        self.assertEqual(data['items'][0]['metadata']['name'], name, data)
+
+    def test_get_pvc(self):
+        # test success
+        name = self.create()
+        response = self.scheduler.pvc.get(self.namespace, name)
+        data = response.json()
+        self.assertEqual(response.status_code, 200, data)
+        # simple verify of data
+        self.assertEqual(data['metadata']['name'], name, data)
+
+    def test_get_pvcs_failure(self):
+        # test failure
+        with self.assertRaises(
+            KubeHTTPException,
+            msg='failed to get Pod doesnotexist in Namespace {}: 404 Not Found'.format(self.namespace)  # noqa
+        ):
+            self.scheduler.pvc.get(self.namespace, 'doesnotexist')
