feat(gateway): unify gateway/route API fields and add JSON schema validation

- Add LazySchemaValidator to validate JSON fields at model level
- Add schema validators for Gateway.ports, Route.rules and Route.parent_refs
- Change GatewaySerializer from Serializer to ModelSerializer
- Add validate_parent_refs and validate methods to RouteSerializer
- Add to_internal_value/to_representation for backend_refs <-> backendRefs conversion
- Change Route.save() to raise ValidationError instead of Http404
- Add Django ValidationError handling in custom_exception_handler
- Add GRPCRoute support to route kind validation
- Add gateway.py and route.py schema definitions
- Update tests to use API calls for consistent serializer behavior

Author: duanhongyi

rootfs/api/exceptions.py

@@ -1,3 +1,4 @@
+from django.core.exceptions import ValidationError
 from django.http import Http404
 import logging
 from rest_framework.exceptions import APIException, status
@@ -36,20 +37,25 @@ def custom_exception_handler(exc, context):
     if isinstance(exc, Http404):
         set_rollback()
         return Response(str(exc), status=status.HTTP_404_NOT_FOUND)
+    # Convert Django ValidationError to DRF 400 response
+    if isinstance(exc, ValidationError):
+        set_rollback()
+        if hasattr(exc, 'message_dict'):
+            return Response(exc.message_dict, status=status.HTTP_400_BAD_REQUEST)
+        elif hasattr(exc, 'messages'):
+            return Response({'non_field_errors': exc.messages}, status=status.HTTP_400_BAD_REQUEST)
+        return Response({'non_field_errors': [str(exc)]}, status=status.HTTP_400_BAD_REQUEST)
     # Call REST framework's default exception handler after specific 404 handling,
     # to get the standard error response.
     response = exception_handler(exc, context)
-    # No response means DRF couldn't handle it
-    # Output a generic 500 in a JSON format
+    # No response means DRF couldn't handle it, output a generic 500 in a JSON format
     if response is None:
         import traceback
         traceback.print_exc()
         logging.exception('Uncaught Exception', exc_info=exc)
         set_rollback()
         return Response({'detail': 'Server Error'}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
-
     # log a few different types of exception instead of using APIException
     if isinstance(exc, (DryccException, ServiceUnavailable, HealthcheckException)):
         logging.exception(exc.__cause__, exc_info=exc)
-
     return response

rootfs/api/models/gateway.py

@@ -3,10 +3,10 @@
 import threading
 from django.db import models
 from django.conf import settings
-from django.http import Http404
+from django.core.exceptions import ValidationError
 
 from api.tasks import send_app_log
-from api.utils import validate_label
+from api.utils import validate_json, validate_label
 from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
 
@@ -19,10 +19,25 @@
 HOSTNAME_PROTOCOLS = TLS_PROTOCOLS + ("HTTP", )
 
 
+class LazySchemaValidator:
+    """Defers schema import until validation time to avoid circular imports."""
+
+    def __init__(self, module_path, attr_name):
+        self.module_path = module_path
+        self.attr_name = attr_name
+
+    def __call__(self, value):
+        mod = __import__(self.module_path, fromlist=[self.attr_name])
+        validate_json(value, schema=getattr(mod, self.attr_name))
+
+
 class Gateway(AuditedModel):
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     name = models.CharField(max_length=63, db_index=True, validators=[validate_label])
-    ports = models.JSONField(default=list)
+    ports = models.JSONField(
+        default=list,
+        validators=[LazySchemaValidator("api.serializers.schemas.gateway", "SCHEMA")],
+    )
 
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this service.
@@ -281,9 +296,15 @@ class Route(AuditedModel):
     kind = models.CharField(max_length=15, choices=[
         (key, '/'.join(value)) for key, value in PROTOCOLS_CHOICES.items()])
     name = models.CharField(max_length=63, db_index=True)
-    rules = models.JSONField(default=list)
+    rules = models.JSONField(
+        default=list,
+        validators=[LazySchemaValidator("api.serializers.schemas.rules", "SCHEMA")],
+    )
     routable = models.BooleanField(default=True)
-    parent_refs = models.JSONField(default=list)
+    parent_refs = models.JSONField(
+        default=list,
+        validators=[LazySchemaValidator("api.serializers.schemas.route", "PARENT_REFS_SCHEMA")],
+    )
 
     @property
     def services(self):
@@ -318,23 +339,6 @@ def cleaned_rules(self):
                 rules.append(rule)
         return rules
 
-    def check_rules(self, rules):
-        for rule in rules:
-            for backend_ref in rule["backendRefs"]:
-                kind = backend_ref.get("kind", "Service")
-                if kind != "Service":
-                    return False, {"detail": "BackendRef only supports service kind"}
-                has_service = False
-                for service in self.services:
-                    ports = [item["port"] for item in service.ports]
-                    if backend_ref["port"] in ports and backend_ref["name"] == service.name:
-                        has_service = True
-                        break
-                if not has_service:
-                    msg = f"service {backend_ref['name']}:{backend_ref['port']} does not exist"
-                    return False, {"detail": msg}
-        return True, ""
-
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this service.
 
@@ -395,12 +399,11 @@ def detach(self, gateway_name, port):
 
     def save(self, *args, **kwargs):
         self.change_default_tls()
-        cleaned_rules = self.cleaned_rules
-        if not cleaned_rules:
+        if not self.cleaned_rules:
             msg = f"route {self.name} no available backend"
             self.log(msg, level=logging.ERROR)
-            raise Http404(msg)
-        self.rules = cleaned_rules
+            raise ValidationError(msg)
+        self.rules = self.cleaned_rules
         super().save(*args, **kwargs)
         self.refresh_to_k8s()
 

rootfs/api/serializers/__init__.py

@@ -26,6 +26,8 @@
 from .schemas.autoscale import SCHEMA as AUTOSCALE_SCHEMA
 from .schemas.healthcheck import SCHEMA as HEALTHCHECK_SCHEMA
 from .schemas.dryccfile import (SCHEMA as DRYCCFILE_SCHEMA, PROCTYPE_REGEX, CONFIGKEY_REGEX)
+from .schemas.gateway import PORT_SCHEMA as GATEWAY_PORT_SCHEMA
+from .schemas.route import PARENT_REF_SCHEMA as ROUTE_PARENT_REF_SCHEMA
 
 
 User = get_user_model()
@@ -36,7 +38,7 @@
 GATEWAY_PROTOCOL_MATCH = re.compile(r'^(HTTP|HTTPS|TCP|TLS|UDP)$')
 GATEWAY_PROTOCOL_MISMATCH_MSG = (
     "the gateway protocol only supports: %s" % GATEWAY_PROTOCOL_MATCH.pattern)
-ROUTE_PROTOCOL_MATCH = re.compile(r'^(HTTPRoute|TCPRoute|UDPRoute|TLSRoute)$')
+ROUTE_PROTOCOL_MATCH = re.compile(r'^(HTTPRoute|TCPRoute|UDPRoute|GRPCRoute|TLSRoute)$')
 ROUTE_PROTOCOL_MISMATCH_MSG = (
     "the route kind only supports: %s" % ROUTE_PROTOCOL_MATCH.pattern)
 PROCTYPE_MATCH = re.compile(PROCTYPE_REGEX)
@@ -735,16 +737,15 @@ def validate(self, attrs):
         return attrs
 
 
-class GatewaySerializer(serializers.Serializer):
+class GatewaySerializer(serializers.ModelSerializer):
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
     name = serializers.CharField(max_length=63, required=True)
-    listeners = serializers.JSONField(required=False)
+    ports = serializers.JSONField(required=True)
     addresses = serializers.JSONField(read_only=True)
 
-    def to_representation(self, instance):
-        representation = super().to_representation(instance)
-        representation['addresses'] = instance.addresses
-        return representation
+    class Meta:
+        model = models.gateway.Gateway
+        fields = '__all__'
 
     validate_name = staticmethod(validate_name)
 
@@ -757,13 +758,35 @@ def validate_protocol(value):
     validate_port = staticmethod(validate_port)
     validate_ptype = staticmethod(validate_ptype)
 
+    @staticmethod
+    def validate_ports(value):
+        if not isinstance(value, list):
+            raise serializers.ValidationError("ports must be a list")
+        for item in value:
+            validate_json(item, GATEWAY_PORT_SCHEMA, serializers.ValidationError)
+        return value
+
+    def validate(self, attrs):
+        attrs = super().validate(attrs)
+        gateway = self.instance or models.gateway.Gateway(**attrs)
+        new_ports = []
+        for item in attrs.get("ports", gateway.ports):
+            port, protocol = item["port"], item["protocol"]
+            gateway.ports = new_ports
+            if not gateway._check_port(port, protocol):
+                raise serializers.ValidationError({"detail": "port is occupied"})
+            new_ports.append({"port": port, "protocol": protocol})
+        attrs["ports"] = new_ports
+        return attrs
+
 
 class RouteSerializer(serializers.ModelSerializer):
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
     kind = serializers.CharField(max_length=15, required=True)
     name = serializers.CharField(max_length=63, required=True)
-    rules = serializers.JSONField(required=False)
-    parent_refs = serializers.JSONField(required=False)
+    rules = serializers.JSONField(required=True)
+    routable = serializers.BooleanField(read_only=True)
+    parent_refs = serializers.JSONField(required=True)
 
     class Meta:
         """Metadata options for a :class:`RouteSerializer`."""
@@ -778,14 +801,35 @@ def validate_kind(value):
             raise serializers.ValidationError(ROUTE_PROTOCOL_MISMATCH_MSG)
         return value
 
-    def validate_rules(self, value):
-        kind = getattr(self, "initial_data", {}).get("kind", None)
+    def validate_rules(self, value, kind=None):
+        if kind is None:
+            kind = getattr(self, "initial_data", {}).get("kind", None)
         if kind:
-            schema = getattr(rules, f"{kind.replace("Route", "")}_RULES_SCHEMA", "SCHEMA")
+            schema = getattr(rules, f"{kind.replace('Route', '')}_RULES_SCHEMA", rules.SCHEMA)
         else:
             schema = rules.SCHEMA
         return validate_json(value, schema, serializers.ValidationError)
 
+    @staticmethod
+    def validate_parent_refs(value):
+        if not isinstance(value, list):
+            raise serializers.ValidationError("parent_refs must be a list")
+        for item in value:
+            validate_json(item, ROUTE_PARENT_REF_SCHEMA, serializers.ValidationError)
+        return value
+
+    def validate(self, attrs):
+        attrs = super().validate(attrs)
+        self.validate_rules(attrs.get("rules"), kind=attrs.get("kind"))
+        route = self.instance or models.gateway.Route(**attrs)
+        if self.instance and self.instance.kind != attrs.get("kind"):
+            raise serializers.ValidationError({"detail": "route kind cannot be changed"})
+        for parent_ref in attrs.get("parent_refs", route.parent_refs):
+            ok, msg = route._check_parent(parent_ref["name"], parent_ref["port"])
+            if not ok:
+                raise serializers.ValidationError(msg)
+        return attrs
+
 
 class WorkspaceSerializer(serializers.ModelSerializer):
     """Serialize Workspace model."""

rootfs/api/serializers/schemas/gateway.py

@@ -0,0 +1,23 @@
+PORT_SCHEMA = {
+    "type": "object",
+    "additionalProperties": False,
+    "required": ["port", "protocol"],
+    "properties": {
+        "port": {
+            "type": "integer",
+            "minimum": 1,
+            "maximum": 65535,
+        },
+        "protocol": {
+            "type": "string",
+            "enum": ["HTTP", "HTTPS", "TCP", "UDP", "TLS"],
+        },
+    },
+}
+
+
+SCHEMA = {
+    "$schema": "http://json-schema.org/schema#",
+    "type": "array",
+    "items": PORT_SCHEMA,
+}

rootfs/api/serializers/schemas/route.py

@@ -0,0 +1,22 @@
+PARENT_REF_SCHEMA = {
+    "type": "object",
+    "additionalProperties": False,
+    "required": ["name", "port"],
+    "properties": {
+        "name": {
+            "type": "string",
+        },
+        "port": {
+            "type": "integer",
+            "minimum": 1,
+            "maximum": 65535,
+        },
+    },
+}
+
+
+PARENT_REFS_SCHEMA = {
+    "$schema": "http://json-schema.org/schema#",
+    "type": "array",
+    "items": PARENT_REF_SCHEMA,
+}