Remove ExtraMetadata in favor of SparseFields

geonode/api/resourcebase_api.py

@@ -142,10 +142,6 @@ def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs):
         if "app_type__in" in filters:
             orm_filters.update({"resource_type": filters["app_type__in"].lower()})
 
-        _metadata = {f"metadata__{_k}": _v for _k, _v in filters.items() if _k.startswith("metadata__")}
-        if _metadata:
-            orm_filters.update({"metadata_filters": _metadata})
-
         if "extent" in filters:
             orm_filters.update({"extent": filters["extent"]})
         orm_filters["f_method"] = filters["f_method"] if "f_method" in filters else "and"
@@ -165,7 +161,6 @@ def apply_filters(self, request, applicable_filters):
         keywords = applicable_filters.pop("keywords__slug__in", None)
         metadata_only = applicable_filters.pop("metadata_only", False)
         filtering_method = applicable_filters.pop("f_method", "and")
-        metadata_filters = applicable_filters.pop("metadata_filters", None)
         if filtering_method == "or":
             filters = Q()
             for f in applicable_filters.items():
@@ -207,9 +202,6 @@ def apply_filters(self, request, applicable_filters):
         if keywords:
             filtered = self.filter_h_keywords(filtered, keywords)
 
-        if metadata_filters:
-            filtered = filtered.filter(**metadata_filters)
-
         # return filtered
         return get_visible_resources(
             filtered,

geonode/api/tests.py

@@ -37,7 +37,6 @@
 from geonode.layers.models import Dataset
 from geonode.documents.models import Document
 from geonode.base.models import (
-    ExtraMetadata,
     Thesaurus,
     ThesaurusLabel,
     ThesaurusKeyword,
@@ -534,37 +533,6 @@ def test_category_filters(self):
         self.assertValidJSONResponse(resp)
         self.assertEqual(len(self.deserialize(resp)["objects"]), 5)
 
-    def test_metadata_filters(self):
-        """Test category filtering"""
-        _r = Dataset.objects.first()
-        _m = ExtraMetadata.objects.create(
-            resource=_r,
-            metadata={
-                "name": "metadata-updated",
-                "slug": "metadata-slug-updated",
-                "help_text": "this is the help text-updated",
-                "field_type": "str-updated",
-                "value": "my value-updated",
-                "category": "category",
-            },
-        )
-
-        list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"})
-        _r.metadata.add(_m)
-        # check we get the correct layers number returnered filtering on one
-        # and then two different categories
-        filter_url = f"{list_url}?metadata__category=category"
-
-        resp = self.api_client.get(filter_url)
-        self.assertValidJSONResponse(resp)
-        self.assertEqual(len(self.deserialize(resp)["objects"]), 1)
-
-        filter_url = f"{list_url}?metadata__category=not-existing-category"
-
-        resp = self.api_client.get(filter_url)
-        self.assertValidJSONResponse(resp)
-        self.assertEqual(len(self.deserialize(resp)["objects"]), 0)
-
     def test_tag_filters(self):
         """Test keywords filtering"""
         list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "datasets"})

geonode/base/api/serializers.py

@@ -27,7 +27,7 @@
 from django.contrib.auth.models import Group
 from django.forms.models import model_to_dict
 from django.contrib.auth import get_user_model
-from django.db.models.query import QuerySet
+
 from geonode.assets.utils import get_default_asset, is_asset_deletable
 from geonode.metadata.multilang.serializers import MultiLangOutputMixin
 from geonode.people import Roles
@@ -57,7 +57,6 @@
     SpatialRepresentationType,
     ThesaurusKeyword,
     ThesaurusKeywordLabel,
-    ExtraMetadata,
     LinkedResource,
 )
 from geonode.documents.models import Document
@@ -282,23 +281,6 @@ def get_attribute(self, instance):
         return build_absolute_uri(instance.detail_url)
 
 
-class ExtraMetadataSerializer(DynamicModelSerializer):
-    class Meta:
-        model = ExtraMetadata
-        name = "ExtraMetadata"
-        fields = ("pk", "metadata")
-
-    def to_representation(self, obj):
-        if isinstance(obj, QuerySet):
-            out = []
-            for el in obj:
-                out.append({**{"id": el.id}, **el.metadata})
-            return out
-        elif isinstance(obj, list):
-            return obj
-        return {**{"id": obj.id}, **obj.metadata}
-
-
 class ThumbnailUrlField(DynamicComputedField):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -672,7 +654,6 @@ class ResourceBaseSerializer(MultiLangOutputMixin, DynamicModelSerializer):
     links = DynamicRelationField(LinksSerializer, source="id", read_only=True)
 
     # Deferred fields
-    metadata = ComplexDynamicRelationField(ExtraMetadataSerializer, many=True, deferred=True)
     data = DataBlobField(DataBlobSerializer, source="id", deferred=True, required=False)
     executions = DynamicRelationField(
         ResourceExecutionRequestSerializer, source="id", deferred=True, required=False, read_only=True
@@ -754,7 +735,6 @@ class Meta:
             "sourcetype",
             "is_copyable",
             "blob",
-            "metadata",
             "executions",
             "linked_resources",
             "download_url",

geonode/base/api/tests.py

@@ -71,7 +71,6 @@
     ResourceBase,
     TopicCategory,
     ThesaurusKeyword,
-    ExtraMetadata,
     RestrictionCodeType,
     License,
     Group,
@@ -3084,77 +3083,6 @@ def test_metadata_uploaded_preserve_can_be_updated(self):
         self.assertTrue(response.json()["resource"]["metadata_uploaded_preserve"])
 
 
-class TestExtraMetadataBaseApi(GeoNodeBaseTestSupport):
-    def setUp(self):
-        self.layer = create_single_dataset("single_layer")
-        self.metadata = {
-            "filter_header": "Foo Filter header",
-            "field_name": "metadata-name",
-            "field_label": "this is the help text",
-            "field_value": "foo",
-        }
-        m = ExtraMetadata.objects.create(resource=self.layer, metadata=self.metadata)
-        self.layer.metadata.add(m)
-        self.mdata = ExtraMetadata.objects.first()
-
-    def test_get_will_return_the_list_of_extra_metadata(self):
-        self.client.login(username="admin", password="admin")
-        url = reverse("base-resources-extra-metadata", args=[self.layer.id])
-        response = self.client.get(url, content_type="application/json")
-        self.assertTrue(200, response.status_code)
-        expected = [{**{"id": self.mdata.id}, **self.metadata}]
-        self.assertEqual(expected, response.json())
-
-    def test_put_will_update_the_whole_metadata(self):
-        self.client.login(username="admin", password="admin")
-        url = reverse("base-resources-extra-metadata", args=[self.layer.id])
-        input_metadata = {
-            "id": self.mdata.id,
-            "filter_header": "Foo Filter header",
-            "field_name": "metadata-updated",
-            "field_label": "this is the help text",
-            "field_value": "foo",
-        }
-        response = self.client.put(url, data=[input_metadata], content_type="application/json")
-        self.assertTrue(200, response.status_code)
-        self.assertEqual([input_metadata], response.json())
-
-    def test_post_will_add_new_metadata(self):
-        self.client.login(username="admin", password="admin")
-        url = reverse("base-resources-extra-metadata", args=[self.layer.id])
-        input_metadata = {
-            "filter_header": "Foo Filter header",
-            "field_name": "metadata-updated",
-            "field_label": "this is the help text",
-            "field_value": "foo",
-        }
-        response = self.client.post(url, data=[input_metadata], content_type="application/json")
-        self.assertTrue(201, response.status_code)
-        self.assertEqual(2, len(response.json()))
-
-    def test_delete_will_delete_single_metadata(self):
-        self.client.login(username="admin", password="admin")
-        url = reverse("base-resources-extra-metadata", args=[self.layer.id])
-        response = self.client.delete(url, data=[self.mdata.id], content_type="application/json")
-        self.assertTrue(200, response.status_code)
-        self.assertEqual([], response.json())
-
-    def test_user_without_view_perms_cannot_see_the_endpoint(self):
-        from geonode.resource.manager import resource_manager
-
-        self.client.login(username="bobby", password="bob")
-        resource_manager.remove_permissions(self.layer.uuid, instance=self.layer.get_self_resource())
-        url = reverse("base-resources-extra-metadata", args=[self.layer.id])
-        response = self.client.get(url, content_type="application/json")
-        self.assertTrue(401, response.status_code)
-
-        perm_spec = {"users": {"bobby": ["view_resourcebase"]}, "groups": {}}
-        self.layer.set_permissions(perm_spec)
-        url = reverse("base-resources-extra-metadata", args=[self.layer.id])
-        response = self.client.get(url, content_type="application/json")
-        self.assertTrue(200, response.status_code)
-
-
 class TestApiLinkedResources(GeoNodeBaseTestSupport):
     @classmethod
     def setUpClass(cls) -> None:

geonode/base/api/views.py

@@ -65,7 +65,6 @@
     TopicCategory,
     ThesaurusKeyword,
     Configuration,
-    ExtraMetadata,
     LinkedResource,
 )
 from geonode.base.api.filters import (
@@ -90,7 +89,6 @@
 from geonode.resource.manager import resource_manager
 
 from .permissions import (
-    IsOwnerOrAdmin,
     IsManagerEditOrAdmin,
     ResourceBasePermissionsFilter,
     UserHasPerms,
@@ -107,12 +105,11 @@
     TopicCategorySerializer,
     RegionSerializer,
     ThesaurusKeywordSerializer,
-    ExtraMetadataSerializer,
     LinkedResourceSerializer,
 )
 from geonode.people.api.serializers import UserSerializer
 from .pagination import GeoNodeApiPagination
-from geonode.base.utils import validate_extra_metadata, patch_perms
+from geonode.base.utils import patch_perms
 from geonode.assets.models import Asset
 from geonode.assets.utils import create_asset_and_link, unlink_asset
 from geonode.assets.handlers import asset_handler_registry
@@ -1282,83 +1279,6 @@ def set_thumbnail(self, request, pk, *args, **kwargs):
             return Response({"thumbnail_url": resource.thumbnail_url})
         return Response("Unable to set thumbnail", status=status.HTTP_400_BAD_REQUEST)
 
-    @extend_schema(
-        methods=["get", "put", "delete", "post"], description="Get/Update/Delete/Add extra metadata for resource"
-    )
-    @action(
-        detail=True,
-        methods=["get", "put", "delete", "post"],
-        permission_classes=[IsOwnerOrAdmin, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}})],
-        url_path=r"extra_metadata",  # noqa
-        url_name="extra-metadata",
-    )
-    def extra_metadata(self, request, pk, *args, **kwargs):
-        _obj = get_object_or_404(ResourceBase, pk=pk)
-
-        if request.method == "GET":
-            # get list of available metadata
-            queryset = _obj.metadata.all()
-            _filters = [{f"metadata__{key}": value} for key, value in request.query_params.items()]
-            if _filters:
-                queryset = queryset.filter(**_filters[0])
-            return Response(ExtraMetadataSerializer().to_representation(queryset))
-        if not request.method == "DELETE":
-            try:
-                extra_metadata = validate_extra_metadata(request.data, _obj)
-            except Exception as e:
-                return Response(status=500, data=e.args[0])
-
-        if request.method == "PUT":
-            """
-            update specific metadata. The ID of the metadata is required to perform the update
-            [
-                {
-                        "id": 1,
-                        "name": "foo_name",
-                        "slug": "foo_sug",
-                        "help_text": "object",
-                        "field_type": "int",
-                        "value": "object",
-                        "category": "object"
-                }
-            ]
-            """
-            for _m in extra_metadata:
-                _id = _m.pop("id")
-                ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id=_id).update(metadata=_m)
-            logger.info("metadata updated for the selected resource")
-            _obj.refresh_from_db()
-            return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all()))
-        elif request.method == "DELETE":
-            # delete single metadata
-            """
-            Expect a payload with the IDs of the metadata that should be deleted. Payload be like:
-            [4, 3]
-            """
-            ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id__in=request.data).delete()
-            _obj.refresh_from_db()
-            return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all()))
-        elif request.method == "POST":
-            # add new metadata
-            """
-            [
-                {
-                        "name": "foo_name",
-                        "slug": "foo_sug",
-                        "help_text": "object",
-                        "field_type": "int",
-                        "value": "object",
-                        "category": "object"
-                }
-            ]
-            """
-            for _m in extra_metadata:
-                new_m = ExtraMetadata.objects.create(resource=_obj, metadata=_m)
-                new_m.save()
-                _obj.metadata.add(new_m)
-            _obj.refresh_from_db()
-            return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all()), status=201)
-
     @action(
         detail=True,
         methods=["get"],

geonode/base/migrations/0098_migrate_extrametadata_to_sparsefields.py

@@ -0,0 +1,52 @@
+import json
+import logging
+
+from django.db import migrations
+
+logger = logging.getLogger(__name__)
+
+
+def migrate_extrametadata_to_sparsefields(apps, schema_editor):
+    """
+    Migrate ExtraMetadata entries to SparseField entries.
+
+    Each ExtraMetadata object (with its JSON dict) is stored as a single
+    SparseField entry with name 'extra_metadata_<pk>' and value as JSON string.
+    Entries whose serialized JSON exceeds 1024 characters are skipped with a warning.
+    """
+    ExtraMetadata = apps.get_model("base", "ExtraMetadata")
+    SparseField = apps.get_model("metadata", "SparseField")
+
+    for extra_meta in ExtraMetadata.objects.select_related("resource").iterator():
+        name = f"extra_metadata_{extra_meta.pk}"
+        value = json.dumps(extra_meta.metadata)
+        if len(value) > 1024:
+            logger.warning(
+                f"ExtraMetadata pk={extra_meta.pk} skipped during migration to SparseField: "
+                f"serialized value exceeds 1024 characters"
+            )
+            continue
+        SparseField.objects.get_or_create(
+            resource=extra_meta.resource,
+            name=name,
+            defaults={"value": value},
+        )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("base", "0097_alter_link_asset"),
+        ("metadata", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_extrametadata_to_sparsefields, migrations.RunPython.noop),
+        migrations.RemoveField(
+            model_name="resourcebase",
+            name="metadata",
+        ),
+        migrations.DeleteModel(
+            name="ExtraMetadata",
+        ),
+    ]