From 7851c0c637a62eb85581dfb050d7b694b1b8e190 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Thu, 21 Mar 2024 11:13:43 +0800 Subject: [PATCH] chore(python): bump python 3.12 --- .../templates/controller-api-deployment.yaml | 8 ---- .../templates/controller-cronjob-daily.yaml | 2 +- .../templates/controller-cronjob-hourly.yaml | 2 +- ...-init.yaml => controller-job-upgrade.yaml} | 25 ++++++++--- rootfs/Dockerfile | 2 +- rootfs/Dockerfile.test | 10 ++--- .../api/management/commands/cluster_lock.py | 6 +-- rootfs/api/models/appsettings.py | 14 +++++- rootfs/api/models/release.py | 19 ++++++-- rootfs/api/models/resource.py | 2 +- rootfs/api/models/service.py | 2 +- rootfs/api/models/tls.py | 13 +++++- rootfs/api/settings/production.py | 19 ++++---- rootfs/api/signals.py | 22 +++++++--- rootfs/api/tests/test_app.py | 40 ++++++++--------- rootfs/api/tests/test_build.py | 2 +- rootfs/api/tests/test_config.py | 8 ++-- rootfs/api/tests/test_domain.py | 2 +- rootfs/api/tests/test_release.py | 14 +++--- rootfs/api/tests/test_resource.py | 2 +- rootfs/api/tests/test_service.py | 11 ++--- rootfs/api/tests/test_volume.py | 4 +- rootfs/bin/boot | 4 -- rootfs/dev_requirements.txt | 6 +-- rootfs/requirements.txt | 44 +++++++++---------- rootfs/scheduler/mock.py | 2 +- rootfs/scheduler/tests/__init__.py | 16 +++---- rootfs/scheduler/tests/test_deployments.py | 37 +++++++--------- rootfs/scheduler/tests/test_hpa.py | 12 +++-- rootfs/scheduler/tests/test_hpa_12_lower.py | 12 +++-- rootfs/scheduler/tests/test_namespaces.py | 16 +++---- rootfs/scheduler/tests/test_nodes.py | 3 +- rootfs/scheduler/tests/test_pods.py | 12 +++-- rootfs/scheduler/tests/test_rc.py | 12 +++-- rootfs/scheduler/tests/test_scheduler.py | 8 ++-- rootfs/scheduler/tests/test_secrets.py | 18 ++++---- rootfs/scheduler/tests/test_services.py | 18 ++++---- 37 files changed, 236 insertions(+), 213 deletions(-) rename charts/controller/templates/{controller-job-init.yaml => controller-job-upgrade.yaml} (68%) diff --git a/charts/controller/templates/controller-api-deployment.yaml b/charts/controller/templates/controller-api-deployment.yaml index 2c87eee85..b20dee6ea 100644 --- a/charts/controller/templates/controller-api-deployment.yaml +++ b/charts/controller/templates/controller-api-deployment.yaml @@ -84,11 +84,3 @@ spec: name: http {{- include "controller.limits" . | indent 8 }} {{- include "controller.envs" . | indent 8 }} - volumeMounts: - - name: controller-config - readOnly: false - mountPath: /etc/controller - volumes: - - name: controller-config - configMap: - name: controller-config \ No newline at end of file diff --git a/charts/controller/templates/controller-cronjob-daily.yaml b/charts/controller/templates/controller-cronjob-daily.yaml index ad48f5596..d97446e5e 100644 --- a/charts/controller/templates/controller-cronjob-daily.yaml +++ b/charts/controller/templates/controller-cronjob-daily.yaml @@ -16,7 +16,7 @@ spec: spec: template: metadata: - labels: + labels: {{- include "common.labels.standard" . | nindent 12 }} app: drycc-controller-conjob spec: restartPolicy: OnFailure diff --git a/charts/controller/templates/controller-cronjob-hourly.yaml b/charts/controller/templates/controller-cronjob-hourly.yaml index 20b3dc681..94c73ff50 100644 --- a/charts/controller/templates/controller-cronjob-hourly.yaml +++ b/charts/controller/templates/controller-cronjob-hourly.yaml @@ -16,7 +16,7 @@ spec: spec: template: metadata: - labels: + labels: {{- include "common.labels.standard" . | nindent 12 }} app: drycc-controller-conjob spec: restartPolicy: OnFailure diff --git a/charts/controller/templates/controller-job-init.yaml b/charts/controller/templates/controller-job-upgrade.yaml similarity index 68% rename from charts/controller/templates/controller-job-init.yaml rename to charts/controller/templates/controller-job-upgrade.yaml index e9d5492e5..f70913f45 100644 --- a/charts/controller/templates/controller-job-init.yaml +++ b/charts/controller/templates/controller-job-upgrade.yaml @@ -1,12 +1,19 @@ apiVersion: batch/v1 kind: Job metadata: - name: drycc-controller-job-init + name: drycc-controller-job-upgrade + labels: + heritage: drycc + annotations: + component.drycc.cc/version: {{ .Values.imageTag }} spec: template: + metadata: + labels: {{- include "common.labels.standard" . | nindent 8 }} + app: drycc-controller-job-upgrade spec: initContainers: - - name: drycc-controller-job-init + - name: drycc-controller-job-upgrade-init image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest imagePullPolicy: {{.Values.imagePullPolicy}} args: @@ -15,10 +22,10 @@ spec: - -u - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL) - -a - - $(DRYCC_REDIS_ADDRS),$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT) - {{- include "controller.envs" . | indent 6 }} + - $(DRYCC_REDIS_ADDRS) + {{- include "controller.envs" . | indent 8 }} containers: - - name: drycc-controller-job-init + - name: drycc-controller-job-upgrade image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}} imagePullPolicy: {{.Values.imagePullPolicy}} {{- if .Values.diagnosticMode.enabled }} @@ -37,6 +44,14 @@ spec: python -u /workspace/manage.py cluster_lock unlock {{- end }} {{- include "controller.envs" . | indent 8 }} + volumeMounts: + - name: controller-config + readOnly: false + mountPath: /etc/controller + volumes: + - name: controller-config + configMap: + name: controller-config restartPolicy: Never backoffLimit: 0 ttlSecondsAfterFinished: 3600 diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 9c702bd34..4f78c36fe 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -4,7 +4,7 @@ FROM registry.drycc.cc/drycc/base:${CODENAME} ENV DRYCC_UID=1001 \ DRYCC_GID=1001 \ DRYCC_HOME_DIR=/workspace \ - PYTHON_VERSION="3.11" + PYTHON_VERSION="3.12" RUN groupadd drycc --gid ${DRYCC_GID} \ && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR} diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test index 042ff5f0c..964038f08 100644 --- a/rootfs/Dockerfile.test +++ b/rootfs/Dockerfile.test @@ -7,11 +7,11 @@ COPY requirements.txt ${DRYCC_HOME_DIR}/requirements.txt COPY dev_requirements.txt ${DRYCC_HOME_DIR}/dev_requirements.txt ENV PGDATA="/opt/drycc/postgresql/data" \ - PYTHON_VERSION="3.11" \ - REDIS_VERSION="7.0.11" \ - RABBITMQ_VERSION="3.12.0" \ - POSTGRES_VERSION="15.3" \ - GOSU_VERSION="1.16" + PYTHON_VERSION="3.12" \ + REDIS_VERSION="7.2.4" \ + RABBITMQ_VERSION="3.12.13" \ + POSTGRES_VERSION="15.5" \ + GOSU_VERSION="1.17" RUN buildDeps='gcc rustc cargo libffi-dev musl-dev openssl'; \ install-packages mercurial ca-certificates git $buildDeps \ diff --git a/rootfs/api/management/commands/cluster_lock.py b/rootfs/api/management/commands/cluster_lock.py index e05a8625a..2b8db23e5 100644 --- a/rootfs/api/management/commands/cluster_lock.py +++ b/rootfs/api/management/commands/cluster_lock.py @@ -20,16 +20,16 @@ def add_arguments(self, parser): ) def lock(self): - cache.set(lock_key, settings.VERSION) + cache.delete(lock_key) print("lock completed!") def unlock(self): - cache.delete(lock_key) + cache.set(lock_key, settings.VERSION) print("unlock completed!") def waitting(self): while True: - version = cache.get("drycc:controller:version") + version = cache.get(lock_key, None) if version != settings.VERSION: print(waitting_init_msg % (version, settings.VERSION)) else: diff --git a/rootfs/api/models/appsettings.py b/rootfs/api/models/appsettings.py index f26e317e7..1021e4ae6 100644 --- a/rootfs/api/models/appsettings.py +++ b/rootfs/api/models/appsettings.py @@ -9,6 +9,7 @@ from .base import UuidAuditedModel User = get_user_model() +logger = logging.getLogger(__name__) class AppSettings(UuidAuditedModel): @@ -35,6 +36,17 @@ def __init__(self, *args, **kwargs): def __str__(self): return "{}-{}".format(self.app.id, str(self.uuid)[:7]) + def log(self, message, level=logging.INFO): + """Logs a message in the context of this application. + + This prefixes log messages with an application "tag" that the customized + drycc-logspout will be on the lookout for. When it's seen, the message-- usually + an application event of some sort like releasing or scaling, will be considered + as "belonging" to the application instead of the controller and will be handled + accordingly. + """ + logger.log(level, "[{}]: {}".format(self.app.id, message)) + def previous(self): """ Return the previous Release to this one. @@ -167,7 +179,7 @@ def _update_fields(self, ignore_update_fields=None): self.delete() raise AlreadyExists("{} changed nothing".format(self.owner)) summary = ' '.join(self.summary) - self.app.log('summary of app setting changes: {}'.format(summary), logging.DEBUG) + self.log('summary of app setting changes: {}'.format(summary), logging.DEBUG) def diff_canaries(self): prev_app_settings = self.previous() diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py index 36e2393d6..17e92be36 100644 --- a/rootfs/api/models/release.py +++ b/rootfs/api/models/release.py @@ -43,6 +43,17 @@ def __str__(self): def image(self): return self.build.image + def log(self, message, level=logging.INFO): + """Logs a message in the context of this application. + + This prefixes log messages with an application "tag" that the customized + drycc-logspout will be on the lookout for. When it's seen, the message-- usually + an application event of some sort like releasing or scaling, will be considered + as "belonging" to the application instead of the controller and will be handled + accordingly. + """ + logger.log(level, "[{}]: {}".format(self.app.id, message)) + def new(self, user, config, build, summary=None, canary=False): """ Create a new application release using the provided Build and Config @@ -69,7 +80,7 @@ def get_port(self): creds = self.get_registry_auth() if self.build.type == "buildpack": - self.app.log( + self.log( 'buildpack type detected. Defaulting to $PORT %s' % DEFAULT_CONTAINER_PORT) return DEFAULT_CONTAINER_PORT @@ -172,7 +183,7 @@ def cleanup_old(self): # noqa Stray pods no longer relevant to the latest release """ latest_version = 'v{}'.format(self.version) - self.app.log( + self.log( 'Cleaning up RSs for releases older than {} (latest)'.format(latest_version), level=logging.DEBUG ) @@ -194,7 +205,7 @@ def cleanup_old(self): # noqa replica_sets_removal.append(current_version) if replica_sets_removal: - self.app.log( + self.log( 'Found the following versions to cleanup: {}'.format(', '.join(replica_sets_removal)), # noqa level=logging.DEBUG ) @@ -260,7 +271,7 @@ def _cleanup_deployment_secrets_and_configs(self, namespace): # http://kubernetes.io/docs/user-guide/labels/#set-based-requirement 'version__notin': versions } - self.app.log('Cleaning up orphaned env var secrets for application {}'.format(namespace), level=logging.DEBUG) # noqa + self.log('Cleaning up orphaned env var secrets for application {}'.format(namespace), level=logging.DEBUG) # noqa secrets = self.scheduler().secret.get(namespace, labels=labels).json()['items'] if not secrets: secrets = [] diff --git a/rootfs/api/models/resource.py b/rootfs/api/models/resource.py index cae0baa17..4b3da53ce 100644 --- a/rootfs/api/models/resource.py +++ b/rootfs/api/models/resource.py @@ -117,7 +117,7 @@ def log(self, message, level=logging.INFO): as "belonging" to the application instead of the controller and will be handled accordingly. """ - logger.log(level, "[{}]: {}".format(self.uuid, message)) + logger.log(level, "[{}]: {}".format(self.app.id, message)) def bind(self, *args, **kwargs): if self.status != "Ready": diff --git a/rootfs/api/models/service.py b/rootfs/api/models/service.py index 6edc36a55..cf74924d3 100644 --- a/rootfs/api/models/service.py +++ b/rootfs/api/models/service.py @@ -132,7 +132,7 @@ def log(self, message, level=logging.INFO): as "belonging" to the application instead of the controller and will be handled accordingly. """ - logger.log(level, "[{}]: {}".format(self.id, message)) + logger.log(level, "[{}]: {}".format(self.app.id, message)) def _namespace(self): return self.app.id diff --git a/rootfs/api/models/tls.py b/rootfs/api/models/tls.py index e5909ec92..36b299082 100644 --- a/rootfs/api/models/tls.py +++ b/rootfs/api/models/tls.py @@ -74,6 +74,17 @@ def _refresh_secret_to_k8s(self): except KubeException as e: raise ServiceUnavailable('Kubernetes secret could not be created') from e + def log(self, message, level=logging.INFO): + """Logs a message in the context of this application. + + This prefixes log messages with an application "tag" that the customized + drycc-logspout will be on the lookout for. When it's seen, the message-- usually + an application event of some sort like releasing or scaling, will be considered + as "belonging" to the application instead of the controller and will be handled + accordingly. + """ + logger.log(level, "[{}]: {}".format(self.app.id, message)) + def refresh_issuer_to_k8s(self): name = namespace = self.app.id try: @@ -113,7 +124,7 @@ def refresh_certificate_to_k8s(self): msg="certificate {} does not exist".format(namespace), level=logging.INFO) self.scheduler().certificate.create(namespace, name, hosts) else: - self.app.log("skip creating certificate, no domain name set", logging.WARNING) + self.log("skip creating certificate, no domain name set", logging.WARNING) else: self.scheduler().certificate.delete(namespace, name, ignore_exception=True) diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py index 6ccc4463d..d4009f4d0 100644 --- a/rootfs/api/settings/production.py +++ b/rootfs/api/settings/production.py @@ -1,7 +1,6 @@ """ Django settings for the Drycc project. """ -from distutils.util import strtobool import sys import uuid import os.path @@ -17,7 +16,7 @@ # A boolean that turns on/off debug mode. # https://docs.djangoproject.com/en/1.11/ref/settings/#debug -DEBUG = bool(strtobool(os.environ.get('DRYCC_DEBUG', 'false'))) +DEBUG = os.environ.get('DRYCC_DEBUG', 'false').lower() == "true" # If set to True, Django's normal exception handling of view functions # will be suppressed, and exceptions will propagate upwards @@ -148,8 +147,8 @@ CSRF_COOKIE_SAMESITE = None SECURE_CONTENT_TYPE_NOSNIFF = True SECURE_BROWSER_XSS_FILTER = True -SESSION_COOKIE_SECURE = bool(strtobool(os.environ.get('SESSION_COOKIE_SECURE', 'false'))) -CSRF_COOKIE_SECURE = bool(strtobool(os.environ.get('CSRF_COOKIE_SECURE', 'false'))) +SESSION_COOKIE_SECURE = os.environ.get('SESSION_COOKIE_SECURE', 'false').lower() == "true" +CSRF_COOKIE_SECURE = os.environ.get('CSRF_COOKIE_SECURE', 'false').lower() == "true" # Honor HTTPS from a trusted proxy @@ -272,7 +271,7 @@ os.environ.get('KUBERNETES_SERVICE_PORT', '443') ) -K8S_API_VERIFY_TLS = bool(strtobool(os.environ.get('K8S_API_VERIFY_TLS', 'true'))) +K8S_API_VERIFY_TLS = os.environ.get('K8S_API_VERIFY_TLS', 'true').lower() == "true" # drycc prometheus url DRYCC_PROMETHEUS_URL = os.environ.get('DRYCC_PROMETHEUS_URL', '') @@ -288,7 +287,6 @@ # True, true, yes, y and more evaluate to True # False, false, no, n and more evaluate to False -# https://docs.python.org/3/distutils/apiref.html?highlight=distutils.util#distutils.util.strtobool # see the above for all available options # # If a user deploys one build with a Procfile but then forgets to in the next one @@ -297,19 +295,18 @@ # If the user has a Procfile in both deploys then processes are scaled up / down as per usual # # By default the process types are scaled down unless this setting is turned on -DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE = bool(strtobool(os.environ.get( - 'DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE', 'true'))) +DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE = os.environ.get( + 'DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE', 'true').lower() == "true" # True, true, yes, y and more evaluate to True # False, false, no, n and more evaluate to False -# https://docs.python.org/3/distutils/apiref.html?highlight=distutils.util#distutils.util.strtobool # see the above for all available options # # If a previous deploy had a Procfile but then the following deploy has no Procfile then it will # result in a 406 - Not Acceptable # Has priority over DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE -DRYCC_DEPLOY_REJECT_IF_PROCFILE_MISSING = bool(strtobool(os.environ.get( - 'DRYCC_DEPLOY_REJECT_IF_PROCFILE_MISSING', 'false'))) +DRYCC_DEPLOY_REJECT_IF_PROCFILE_MISSING = os.environ.get( + 'DRYCC_DEPLOY_REJECT_IF_PROCFILE_MISSING', 'false').lower() == "true" # Define a global default on how many pods to bring up and then # take down sequentially during a deploy diff --git a/rootfs/api/signals.py b/rootfs/api/signals.py index 1e67341b0..bc96750f0 100644 --- a/rootfs/api/signals.py +++ b/rootfs/api/signals.py @@ -40,7 +40,9 @@ def _log_instance_created(**kwargs): if kwargs.get('created'): instance = kwargs['instance'] message = '{} {} created'.format(instance.__class__.__name__.lower(), instance) - if hasattr(instance, 'app'): + if hasattr(instance, 'log'): + instance.log(message) + elif hasattr(instance, 'app'): instance.app.log(message) else: logger.info(message) @@ -50,7 +52,9 @@ def _log_instance_added(**kwargs): if kwargs.get('created'): instance = kwargs['instance'] message = '{} {} added'.format(instance.__class__.__name__.lower(), instance) - if hasattr(instance, 'app'): + if hasattr(instance, 'log'): + instance.log(message) + elif hasattr(instance, 'app'): instance.app.log(message) else: logger.info(message) @@ -59,7 +63,9 @@ def _log_instance_added(**kwargs): def _log_instance_updated(**kwargs): instance = kwargs['instance'] message = '{} {} updated'.format(instance.__class__.__name__.lower(), instance) - if hasattr(instance, 'app'): + if hasattr(instance, 'log'): + instance.log(message) + elif hasattr(instance, 'app'): instance.app.log(message) else: logger.info(message) @@ -68,7 +74,9 @@ def _log_instance_updated(**kwargs): def _log_instance_removed(**kwargs): instance = kwargs['instance'] message = '{} {} removed'.format(instance.__class__.__name__.lower(), instance) - if hasattr(instance, 'app'): + if hasattr(instance, 'log'): + instance.log(message) + elif hasattr(instance, 'app'): instance.app.log(message) else: logger.info(message) @@ -79,7 +87,7 @@ def _hook_release_created(**kwargs): if kwargs.get('created'): release = kwargs['instance'] # append release lifecycle logs to the app - release.app.log(release.summary) + release.log(release.summary) for deploy_hook in settings.DRYCC_DEPLOY_HOOK_URLS: url = deploy_hook @@ -108,9 +116,9 @@ def _hook_release_created(**kwargs): try: get_session().post(url, headers=headers) # just notify with the base URL, disregard the added URL query - release.app.log('Deploy hook sent to {}'.format(deploy_hook)) + release.log('Sent deploy hook to {}'.format(deploy_hook)) except requests.RequestException as e: - release.app.log('An error occurred while sending the deploy hook to {}: {}'.format( + release.log('An error occurred while sending the deploy hook to {}: {}'.format( deploy_hook, e), logging.ERROR) diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py index 139b6785a..5930ca757 100644 --- a/rootfs/api/tests/test_app.py +++ b/rootfs/api/tests/test_app.py @@ -96,7 +96,7 @@ def test_response_data(self, mock_requests): 'owner': self.user.username, 'structure': {} } - self.assertDictContainsSubset(expected, response.data) + self.assertEqual(response.data, expected | response.data) def test_app_override_id(self, mock_requests): app_id = self.create_app() @@ -107,19 +107,20 @@ def test_app_override_id(self, mock_requests): @mock.patch('api.models.app.logger') def test_app_release_notes_in_logs(self, mock_requests, mock_logger): """Verifies that an app's release summary is dumped into the logs.""" - app_id = self.create_app() - app = App.objects.get(id=app_id) - - # check app logs - exp_msg = "[{app_id}]: {self.user.username} created initial release".format(**locals()) - mock_logger.log.has_calls(logging.INFO, exp_msg) - app.log('hello world') - exp_msg = "[{app_id}]: hello world".format(**locals()) - mock_logger.log.has_calls(logging.INFO, exp_msg) - app.log('goodbye world', logging.WARNING) - # assert logging with a different log level - exp_msg = "[{app_id}]: goodbye world".format(**locals()) - mock_logger.log.has_calls(logging.WARNING, exp_msg) + with mock.patch('api.models.release.logger') as release_logger: + app_id = self.create_app() + app = App.objects.get(id=app_id) + # check release logs + exp_msg = "[{app_id}]: {self.user.username} created initial release".format( + **locals()) + release_logger.log.assert_any_call(logging.INFO, exp_msg) + app.log('hello world') + exp_msg = "[{app_id}]: hello world".format(**locals()) + mock_logger.log.assert_any_call(logging.INFO, exp_msg) + app.log('goodbye world', logging.WARNING) + # assert logging with a different log level + exp_msg = "[{app_id}]: goodbye world".format(**locals()) + mock_logger.log.assert_any_call(logging.WARNING, exp_msg) def test_app_errors(self, mock_requests): response = self.client.post('/v2/apps', {'id': 'camelCase'}) @@ -182,12 +183,13 @@ def test_app_structure_is_valid_json(self, mock_requests): self.assertIn('structure', response.data) self.assertEqual(response.data['structure'], {"web": 1}) - @mock.patch('api.models.app.logger') + @mock.patch('api.models.release.logger') def test_admin_can_manage_other_apps(self, mock_requests, mock_logger): """Administrators of Drycc should be able to manage all applications. """ # log in as non-admin user and create an app - user = User.objects.get(username='autotest2') + username = 'autotest2' + user = User.objects.get(username=username) token = Token.objects.get(user=user).key self.client.credentials(HTTP_AUTHORIZATION='Token ' + token) app_id = self.create_app() @@ -198,10 +200,8 @@ def test_admin_can_manage_other_apps(self, mock_requests, mock_logger): response = self.client.get(url) self.assertEqual(response.status_code, 200, response.data) # check app logs - exp_msg = "autotest2 created initial release" - exp_log_call = mock.call(logging.INFO, exp_msg) - mock_logger.log.has_calls(exp_log_call) - + exp_msg = "[%s]: %s created initial release" % (app_id, username) + mock_logger.log.assert_any_call(logging.INFO, exp_msg) # TODO: test run needs an initial build # delete the app url = '/v2/apps/{}'.format(app_id) diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py index dc12d176d..1cb2c246a 100644 --- a/rootfs/api/tests/test_build.py +++ b/rootfs/api/tests/test_build.py @@ -102,7 +102,7 @@ def test_response_data(self, mock_requests): 'procfile': {}, 'sha': '' } - self.assertDictContainsSubset(expected, response.data) + self.assertEqual(response.data, expected | response.data) def test_build_default_containers(self, mock_requests): app_id = self.create_app() diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py index 532dfb1e2..4ae804976 100644 --- a/rootfs/api/tests/test_config.py +++ b/rootfs/api/tests/test_config.py @@ -145,7 +145,7 @@ def test_default_tags(self, mock_requests): 'tags': {'ssd': 'true'}, 'registry': {} } - self.assertDictContainsSubset(expected, response.data) + self.assertEqual(response.data, expected | response.data) # make sure changes not drop tags body = {'values': json.dumps({'PORT': '5001'})} @@ -161,7 +161,7 @@ def test_default_tags(self, mock_requests): 'tags': {'ssd': 'true'}, 'registry': {} } - self.assertDictContainsSubset(expected, response.data) + self.assertEqual(response.data, response.data | expected) def test_response_data(self, mock_requests): """Test that the serialized response contains only relevant data.""" @@ -187,7 +187,7 @@ def test_response_data(self, mock_requests): 'tags': {}, 'registry': {} } - self.assertDictContainsSubset(expected, response.data) + self.assertEqual(response.data, response.data | expected) def test_response_data_types_converted(self, mock_requests): """Test that config data is converted into the correct type.""" @@ -218,7 +218,7 @@ def test_response_data_types_converted(self, mock_requests): 'tags': {}, 'registry': {} } - self.assertDictContainsSubset(expected, response.data) + self.assertEqual(response.data, expected | response.data) body = {'limits': {PROCFILE_TYPE_WEB: "not-exist"}} response = self.client.post(url, body) diff --git a/rootfs/api/tests/test_domain.py b/rootfs/api/tests/test_domain.py index b0200d3ff..635770b81 100644 --- a/rootfs/api/tests/test_domain.py +++ b/rootfs/api/tests/test_domain.py @@ -51,7 +51,7 @@ def test_response_data(self): 'app': app_id, 'domain': 'test-domain.example.com' } - self.assertDictContainsSubset(expected, response.data) + self.assertEqual(response.data, expected | response.data) def test_strip_dot(self): """Test that a dot on the right side of the domain gets stripped""" diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py index 9df946cac..f1f3a2ad4 100644 --- a/rootfs/api/tests/test_release.py +++ b/rootfs/api/tests/test_release.py @@ -124,7 +124,7 @@ def test_response_data(self, mock_requests): 'summary': '{} added NEW_URL'.format(self.user.username), 'version': 2 } - self.assertDictContainsSubset(expected, response.data) + self.assertEqual(response.data, response.data | expected) def test_release_rollback(self, mock_requests): app_id = self.create_app() @@ -402,7 +402,7 @@ def test_deploy_hooks_logged(self, mock_requests, mock_logger): self.create_app(app_id) # check app logs exp_msg = "[{app_id}]: Sent deploy hook to http://drycc.rocks".format(**locals()) - mock_logger.log.has_calls(logging.INFO, exp_msg) + mock_logger.log.assert_any_call(logging.INFO, exp_msg) self.assertTrue(mr_rocks.called) self.assertEqual(mr_rocks.call_count, 1) @@ -417,11 +417,11 @@ def test_deploy_hooks_logged(self, mock_requests, mock_logger): # check app logs exp_msg = "[{app_id}]: Sent deploy hook to http://drycc.ninja".format(**locals()) - mock_logger.log.has_calls(logging.INFO, exp_msg) + mock_logger.log.assert_any_call(logging.INFO, exp_msg) self.assertTrue(mr_ninja.called) self.assertEqual(mr_ninja.call_count, 1) exp_msg = "[{app_id}]: Sent deploy hook to http://cat.dog".format(**locals()) - mock_logger.log.has_calls(logging.INFO, exp_msg) + mock_logger.log.assert_any_call(logging.INFO, exp_msg) self.assertTrue(mr_catdog.called) self.assertEqual(mr_catdog.call_count, 1) sha = '2345678' @@ -437,11 +437,11 @@ def raise_callback(request, context): # check app logs exp_msg = "[{app_id}]: An error occurred while sending the deploy hook to http://cat.ninja: poop".format(**locals()) # noqa - mock_logger.log.has_calls(logging.ERROR, exp_msg) + mock_logger.log.assert_any_call(logging.ERROR, exp_msg) self.assertTrue(mr_ninja.called) self.assertEqual(mr_ninja.call_count, 1) exp_msg = "[{app_id}]: Sent deploy hook to http://drycc.dog".format(**locals()) - mock_logger.log.has_calls(logging.INFO, exp_msg) + mock_logger.log.assert_any_call(logging.INFO, exp_msg) self.assertTrue(mr_catdog.called) self.assertEqual(mr_catdog.call_count, 1) @@ -477,7 +477,7 @@ def raise_callback(request, context): # check app logs exp_msg = "[{app_id}]: Sent deploy hook to {hook_url}".format(**locals()) - mock_logger.log.has_calls(logging.INFO, exp_msg) + mock_logger.log.assert_any_call(logging.INFO, exp_msg) self.assertTrue(mr_terminator.called) self.assertEqual(mr_terminator.call_count, 1) diff --git a/rootfs/api/tests/test_resource.py b/rootfs/api/tests/test_resource.py index 518df80d5..505d76eb6 100644 --- a/rootfs/api/tests/test_resource.py +++ b/rootfs/api/tests/test_resource.py @@ -54,7 +54,7 @@ def test_resources_create(self, mock_requests): 'name': 'mysql', 'plan': 'mysql:5.6' } - self.assertDictContainsSubset(expected, response.data) + self.assertEqual(response.data, expected | response.data) def test_resources_list(self, mock_requests): """ diff --git a/rootfs/api/tests/test_service.py b/rootfs/api/tests/test_service.py index f5a159bac..d85ef90f2 100644 --- a/rootfs/api/tests/test_service.py +++ b/rootfs/api/tests/test_service.py @@ -62,7 +62,7 @@ def test_service_basic_ops(self): }], "procfile_type": "test" } - self.assertDictContainsSubset(expected0, response.data['services'][0]) + self.assertEqual(response.data['services'][0], expected0 | response.data['services'][0]) # port is occupied response = self.client.post( '/v2/apps/{}/services'.format(app_id), @@ -107,7 +107,8 @@ def test_service_basic_ops(self): self.assertEqual(response.status_code, 204, response.data) response = self.client.get('/v2/apps/{}/services'.format(app_id)) self.assertEqual(response.status_code, 200, response.data) - self.assertDictContainsSubset(expected1, response.data['services'][0]) + self.assertEqual( + response.data['services'][0], expected1 | response.data['services'][0]) # create 2nd service response = self.client.post( @@ -135,15 +136,15 @@ def test_service_basic_ops(self): }], "procfile_type": "test2" } - self.assertDictContainsSubset(expected2, response.data['services'][0]) - self.assertDictContainsSubset(expected1, response.data['services'][1]) + self.assertEqual(response.data['services'][0], expected2 | response.data['services'][0]) + self.assertEqual(response.data['services'][1], expected1 | response.data['services'][1]) # delete port response = self.client.delete( '/v2/apps/{}/services'.format(app_id), {'procfile_type': 'test', "protocol": "TCP", "port": 6000} ) response = self.client.get('/v2/apps/{}/services'.format(app_id)) - self.assertDictContainsSubset(expected0, response.data['services'][1]) + self.assertEqual(response.data['services'][1], expected0 | response.data['services'][1]) # delete 1st response = self.client.delete( '/v2/apps/{}/services'.format(app_id), diff --git a/rootfs/api/tests/test_volume.py b/rootfs/api/tests/test_volume.py index a088e8c29..265a94716 100644 --- a/rootfs/api/tests/test_volume.py +++ b/rootfs/api/tests/test_volume.py @@ -74,7 +74,7 @@ def test_volumecreate(self, mock_requests): 'name': 'myvolume', 'size': '500G' } - self.assertDictContainsSubset(expected, response.data) + self.assertEqual(response.data, expected | response.data) def test_volume_list_unmount(self, mock_requests): """ @@ -122,7 +122,7 @@ def test_volume_expand(self, mock_requests): 'size': '1024G' } assert len(response.data["results"]) == 1 - self.assertDictContainsSubset(expected, response.data["results"][0]) + self.assertEqual(response.data["results"][0], expected | response.data["results"][0]) # nfs expand response = self.client.post( '/v2/apps/{}/volumes'.format(app_id), diff --git a/rootfs/bin/boot b/rootfs/bin/boot index 6cd975d70..d1c2bf08f 100755 --- a/rootfs/bin/boot +++ b/rootfs/bin/boot @@ -23,10 +23,6 @@ echo "" echo "Health Checks:" python /workspace/manage.py healthchecks -echo "" -echo "Waiting for init" -python /workspace/manage.py cluster_lock waitting - # spawn a gunicorn server in the background echo "" echo "Starting up Gunicorn" diff --git a/rootfs/dev_requirements.txt b/rootfs/dev_requirements.txt index 058b80601..5e6b68009 100644 --- a/rootfs/dev_requirements.txt +++ b/rootfs/dev_requirements.txt @@ -1,13 +1,13 @@ # test module # test # Run "make test-unit" for the % of code exercised during tests -coverage==7.2.5 +coverage==7.4.4 # Run "make test-style" to check python syntax and style -flake8==6.0.0 +flake8==7.0.0 # code coverage report at https://codecov.io/github/drycc/controller codecov==2.1.13 # mock out python-requests, mostly k8s -requests-mock==1.10.0 +requests-mock==1.11.0 diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt index 885f3e022..2e36e2989 100644 --- a/rootfs/requirements.txt +++ b/rootfs/requirements.txt @@ -1,31 +1,31 @@ # Drycc controller requirements backoff==2.2.1 -django==4.2.10 +django==4.2.11 channels==4.0.0 -aiohttp==3.9.2 -django-cors-headers==4.1.0 +aiohttp==3.9.3 +django-cors-headers==4.3.1 django-guardian==2.4.0 -djangorestframework==3.14.0 -docker==6.1.3 -gunicorn==20.1.0 -uvicorn[standard]==0.22.0 -asgiref==3.7.2 -idna==3.4 -jsonschema==4.18.0 +djangorestframework==3.15.0 +docker==7.0.0 +gunicorn==21.2.0 +uvicorn[standard]==0.29.0 +asgiref==3.8.0 +idna==3.6 +jsonschema==4.21.1 morph==0.1.5 -packaging==23.1 -pyasn1==0.5.0 -psycopg[binary]==3.1.9 -pyOpenSSL==23.2.0 +packaging==24.0 +pyasn1==0.5.1 +psycopg[binary]==3.1.18 +pyOpenSSL==24.1.0 ndg-httpsclient==0.5.1 -pytz==2023.3 +pytz==2024.1 requests==2.31.0 requests-toolbelt==1.0.0 -celery==5.3.1 -hiredis==2.2.3 -django_redis==5.3.0 -dj-database-url==2.0.0 -social-auth-app-django==5.2.0 +celery==5.3.6 +hiredis==2.3.2 +django_redis==5.4.0 +dj-database-url==2.1.0 +social-auth-app-django==5.4.0 python-jose==3.3.0 -Authlib==1.2.1 -kubernetes==26.1.0 +Authlib==1.3.0 +kubernetes==29.0.0 diff --git a/rootfs/scheduler/mock.py b/rootfs/scheduler/mock.py index b1ce0bb4c..1f11f1c9e 100644 --- a/rootfs/scheduler/mock.py +++ b/rootfs/scheduler/mock.py @@ -49,7 +49,7 @@ def wrapped(*args, **kargs): return wrapped def __enter__(self): - if not type(self.key) == str and self.key == '': + if type(self.key) is not str and self.key == '': raise RuntimeError("Key not specified!") if not self.acquire(self.block): diff --git a/rootfs/scheduler/tests/__init__.py b/rootfs/scheduler/tests/__init__.py index 11db26bd9..4e131397d 100644 --- a/rootfs/scheduler/tests/__init__.py +++ b/rootfs/scheduler/tests/__init__.py @@ -24,14 +24,12 @@ def create_namespace(self): data = response.json() self.assertEqual(data['apiVersion'], 'v1') self.assertEqual(data['kind'], 'Namespace') - self.assertDictContainsSubset( - { - 'name': namespace, - 'labels': { - 'heritage': 'drycc' - } - }, - data['metadata'] - ) + metadata = { + 'name': namespace, + 'labels': { + 'heritage': 'drycc' + } + } + self.assertEqual(data['metadata'], metadata | data['metadata']) return namespace diff --git a/rootfs/scheduler/tests/test_deployments.py b/rootfs/scheduler/tests/test_deployments.py index b8141ae80..6541f226c 100644 --- a/rootfs/scheduler/tests/test_deployments.py +++ b/rootfs/scheduler/tests/test_deployments.py @@ -213,13 +213,11 @@ def test_get_deployment(self): self.assertEqual(data['apiVersion'], 'apps/v1') self.assertEqual(data['kind'], 'Deployment') self.assertEqual(data['metadata']['name'], name) - self.assertDictContainsSubset( - { - 'app': self.namespace, - 'heritage': 'drycc' - }, - data['metadata']['labels'] - ) + labels = { + 'app': self.namespace, + 'heritage': 'drycc' + } + self.assertEqual(data['metadata']['labels'], data['metadata']['labels'] | labels) def test_scale(self): name = self.scale() @@ -288,14 +286,11 @@ def test_get_deployment_replicaset(self): self.assertEqual(data['apiVersion'], 'apps/v1', data) self.assertEqual(data['kind'], 'ReplicaSet', data) self.assertEqual(data['metadata']['name'], replica_name, data) - self.assertDictContainsSubset( - { - 'app': self.namespace, - 'heritage': 'drycc' - }, - data['metadata']['labels'], - data - ) + labels = { + 'app': self.namespace, + 'heritage': 'drycc' + } + self.assertEqual(data['metadata']['labels'], data['metadata']['labels'] | labels, data) def test_get_deployment_annotations(self): """ @@ -307,12 +302,12 @@ def test_get_deployment_annotations(self): } deployment = self.create(**kwargs) data = self.scheduler.deployment.get(self.namespace, deployment).json() - - self.assertDictContainsSubset( - { - 'iam.amazonaws.com/role': 'role-arn' - }, - data['spec']['template']['metadata']['annotations'] + annotations = { + 'iam.amazonaws.com/role': 'role-arn' + } + self.assertEqual( + data['spec']['template']['metadata']['annotations'], + data['spec']['template']['metadata']['annotations'] | annotations, ) def test_get_pod_annotations(self): diff --git a/rootfs/scheduler/tests/test_hpa.py b/rootfs/scheduler/tests/test_hpa.py index 665f3900e..97e93142f 100644 --- a/rootfs/scheduler/tests/test_hpa.py +++ b/rootfs/scheduler/tests/test_hpa.py @@ -211,10 +211,8 @@ def test_get_horizontalpodautoscaler(self): self.assertEqual(data['apiVersion'], 'autoscaling/v1') self.assertEqual(data['kind'], 'HorizontalPodAutoscaler') self.assertEqual(data['metadata']['name'], name) - self.assertDictContainsSubset( - { - 'app': self.namespace, - 'heritage': 'drycc' - }, - data['metadata']['labels'] - ) + labels = { + 'app': self.namespace, + 'heritage': 'drycc' + } + self.assertEqual(data['metadata']['labels'], data['metadata']['labels'] | labels) diff --git a/rootfs/scheduler/tests/test_hpa_12_lower.py b/rootfs/scheduler/tests/test_hpa_12_lower.py index 7d98ba5f7..b6ccb4123 100644 --- a/rootfs/scheduler/tests/test_hpa_12_lower.py +++ b/rootfs/scheduler/tests/test_hpa_12_lower.py @@ -215,10 +215,8 @@ def test_get_horizontalpodautoscaler(self): self.assertEqual(data['apiVersion'], 'autoscaling/v1') self.assertEqual(data['kind'], 'HorizontalPodAutoscaler') self.assertEqual(data['metadata']['name'], name) - self.assertDictContainsSubset( - { - 'app': self.namespace, - 'heritage': 'drycc' - }, - data['metadata']['labels'] - ) + labels = { + 'app': self.namespace, + 'heritage': 'drycc' + } + self.assertEqual(data['metadata']['labels'], labels | data['metadata']['labels']) diff --git a/rootfs/scheduler/tests/test_namespaces.py b/rootfs/scheduler/tests/test_namespaces.py index 33f17adc4..7a2c098aa 100644 --- a/rootfs/scheduler/tests/test_namespaces.py +++ b/rootfs/scheduler/tests/test_namespaces.py @@ -36,15 +36,13 @@ def test_get_namespace(self): self.assertEqual(response.status_code, 200, data) self.assertEqual(data['apiVersion'], 'v1') self.assertEqual(data['kind'], 'Namespace') - self.assertDictContainsSubset( - { - 'name': self.namespace, - 'labels': { - 'heritage': 'drycc' - } - }, - data['metadata'] - ) + metadata = { + 'name': self.namespace, + 'labels': { + 'heritage': 'drycc' + } + } + self.assertEqual(data['metadata'], data['metadata'] | metadata) def test_delete_failure(self): # test failure diff --git a/rootfs/scheduler/tests/test_nodes.py b/rootfs/scheduler/tests/test_nodes.py index 6dd2bd018..adcdec1a1 100644 --- a/rootfs/scheduler/tests/test_nodes.py +++ b/rootfs/scheduler/tests/test_nodes.py @@ -34,4 +34,5 @@ def test_get_node(self): self.assertEqual(data['apiVersion'], 'v1') self.assertEqual(data['kind'], 'Node') self.assertEqual(data['metadata']['name'], name) - self.assertDictContainsSubset({'ssd': 'true'}, data['metadata']['labels']) + labels = {'ssd': 'true'} + self.assertEqual(data['metadata']['labels'], data['metadata']['labels'] | labels) diff --git a/rootfs/scheduler/tests/test_pods.py b/rootfs/scheduler/tests/test_pods.py index 864b6780d..4774035a6 100644 --- a/rootfs/scheduler/tests/test_pods.py +++ b/rootfs/scheduler/tests/test_pods.py @@ -86,13 +86,11 @@ def test_get_pod(self): self.assertEqual(response.status_code, 200, data) self.assertEqual(data['kind'], 'Pod') self.assertEqual(data['metadata']['name'], name) - self.assertDictContainsSubset( - { - 'app': self.namespace, - 'heritage': 'drycc' - }, - data['metadata']['labels'] - ) + labels = { + 'app': self.namespace, + 'heritage': 'drycc' + } + self.assertEqual(data['metadata']['labels'], data['metadata']['labels'] | labels) def test_liveness_status(self): # Missing Ready type means pod has passed liveness check diff --git a/rootfs/scheduler/tests/test_rc.py b/rootfs/scheduler/tests/test_rc.py index 97f92adb9..efe0f2c23 100644 --- a/rootfs/scheduler/tests/test_rc.py +++ b/rootfs/scheduler/tests/test_rc.py @@ -126,10 +126,8 @@ def test_get_rc(self): self.assertEqual(data['apiVersion'], 'v1') self.assertEqual(data['kind'], 'ReplicationController') self.assertEqual(data['metadata']['name'], name) - self.assertDictContainsSubset( - { - 'app': self.namespace, - 'heritage': 'drycc' - }, - data['metadata']['labels'] - ) + labels = { + 'app': self.namespace, + 'heritage': 'drycc' + } + self.assertEqual(data['metadata']['labels'], data['metadata']['labels'] | labels) diff --git a/rootfs/scheduler/tests/test_scheduler.py b/rootfs/scheduler/tests/test_scheduler.py index fc2769f9e..fcfb1176f 100644 --- a/rootfs/scheduler/tests/test_scheduler.py +++ b/rootfs/scheduler/tests/test_scheduler.py @@ -35,7 +35,7 @@ def test_set_container_applies_healthcheck_with_routable(self): self.scheduler.pod._set_container( 'foo', 'bar', data, routable=True, healthcheck=healthcheck ) - self.assertDictContainsSubset(healthcheck, data) + self.assertEqual(data, data | healthcheck) data = {} self.scheduler.pod._set_container( 'foo', 'bar', data, routable=True, build_type="buildpack", healthcheck={} @@ -56,7 +56,7 @@ def test_set_container_applies_healthcheck_with_routable(self): 'foo', 'bar', data, routable=False, healthcheck=healthcheck ) - self.assertDictContainsSubset(healthcheck, data) + self.assertEqual(data, data | healthcheck) self.assertEqual(data.get('readinessProbe'), None) # now call without setting 'routable', should default to False @@ -64,7 +64,7 @@ def test_set_container_applies_healthcheck_with_routable(self): self.scheduler.pod._set_container( 'foo', 'bar', data, healthcheck=healthcheck ) - self.assertDictContainsSubset(healthcheck, data) + self.assertEqual(data, data | healthcheck) self.assertEqual(data.get('readinessProbe'), None) data = {} @@ -78,7 +78,7 @@ def test_set_container_applies_healthcheck_with_routable(self): self.scheduler.pod._set_health_checks( data, {'PORT': 80}, healthcheck=livenessProbe ) - self.assertDictContainsSubset(healthcheck, data) + self.assertEqual(data, data | healthcheck) self.assertEqual(data.get('readinessProbe'), None) def test_set_container_limits(self): diff --git a/rootfs/scheduler/tests/test_secrets.py b/rootfs/scheduler/tests/test_secrets.py index faaddedec..668c0adc1 100644 --- a/rootfs/scheduler/tests/test_secrets.py +++ b/rootfs/scheduler/tests/test_secrets.py @@ -114,16 +114,14 @@ def test_get_secret(self): self.assertEqual(response.status_code, 200, data) self.assertEqual(data['apiVersion'], 'v1') self.assertEqual(data['kind'], 'Secret') - self.assertDictContainsSubset( - { - 'name': name, - 'labels': { - 'app': self.namespace, - 'heritage': 'drycc' - } - }, - data['metadata'] - ) + metadata = { + 'name': name, + 'labels': { + 'app': self.namespace, + 'heritage': 'drycc' + } + } + self.assertEqual(data['metadata'], data['metadata'] | metadata) self.assertEqual(data['data']['foo'], 'bar', data) self.assertEqual(data['data']['this'], 'that', data) self.assertEqual(data['type'], 'Opaque') diff --git a/rootfs/scheduler/tests/test_services.py b/rootfs/scheduler/tests/test_services.py index e769a954b..f9a739fbd 100644 --- a/rootfs/scheduler/tests/test_services.py +++ b/rootfs/scheduler/tests/test_services.py @@ -128,14 +128,12 @@ def test_get_service(self): self.assertEqual(response.status_code, 200, data) self.assertEqual(data['apiVersion'], 'v1') self.assertEqual(data['kind'], 'Service') - self.assertDictContainsSubset( - { - 'name': name, - 'labels': { - 'app': self.namespace, - 'heritage': 'drycc' - } - }, - data['metadata'] - ) + metadata = { + 'name': name, + 'labels': { + 'app': self.namespace, + 'heritage': 'drycc' + } + } + self.assertEqual(data['metadata'], data['metadata'] | metadata) self.assertEqual(data['spec']['ports'][0]['targetPort'], 5000)