[ENG-5493][ENG-5412] more helpful trove responses (#820)

[ENG-5493]
- fill in `trove/exceptions.py` with situation-specific exception classes
- use those specific exceptions thruout trove code
- give an appropriate response when those specific exceptions happen while handling a request (currently always in jsonapi)
- reduce dead links and other errors when browsing the search api
  - allow resolving static vocab terms (including "new" static vocabs `skos`, `foaf`, `dc`, `dctype`, `dcam`)
  - display datatypes for literals
- add better search-api tests (with actual search indexes running)
- fix broken `next` links with field-specific search ([ENG-5412])
- code cleanup
  - add `trove.trovesearch` with modules previously at `trove.trovesearch_gathering`, `share.search.search_params`, and `share.search.search_response`
  - render derived indexcard metadata as rdf literal (except html rendering) -- avoid errors from non-standard "quoted triple" or "quoted graph"
  - remove `trove.render.osfmap_jsonld` (subsumed by `trove.derive.osfmap_json`)
  - remove `ingest_rdf_vocabs` management command (unnecessary with "static vocab" update)
  - rename `...VOCAB` to `...THESAURUS` (not just a set of terms, but a whole rdf graph describing those terms)
  - remove `trove.util.iri_labeler` (completely replaced by `primitive_rdf.IriShorthand`
  - consolidate mediatypes as constants in `trove.vocab.mediatypes`
  - move all "static vocabs" into `trove.vocab.static_vocab`
  - add `functools.cache` to things that need be computed only once

requirements.txt

@@ -31,7 +31,7 @@ psycogreen==1.0.2  # BSD
 psycopg2==2.9.5  # LGPL with exceptions or ZPL
 python-dateutil==2.8.1  # Apache 2.0
 PyJWE==1.0.0 # Apache 2.0
-pyshacl==0.22.0 # Apache 2.0
+rdflib==7.0.0
 pyyaml==6.0  # MIT
 requests==2.25.1  # Apache 2.0
 sentry-sdk[django]==1.22.2  # MIT
@@ -43,4 +43,4 @@ xmltodict==0.12.0  # MIT
 # Allows custom-rendered IDs, hiding null values, and including data in error responses
 git+https://github.com/cos-forks/[email protected]+cos0
 
-git+https://github.com/aaxelb/primitive_metadata.git@0.2023.57
+git+https://github.com/aaxelb/primitive_metadata.git@0.2024.09

share/search/exceptions.py

@@ -15,15 +15,3 @@ class DaemonIndexingError(ShareException):
 
 class IndexStrategyError(ShareException):
     pass
-
-
-class SearchApiError(ShareException):
-    pass
-
-
-class InvalidSearchParam(SearchApiError):
-    pass
-
-
-class UnsupportedSearchParam(IndexStrategyError):
-    pass

share/search/index_strategy/_base.py

@@ -10,15 +10,15 @@
 from share.models.index_backfill import IndexBackfill
 from share.search.exceptions import IndexStrategyError
 from share.search.index_status import IndexStatus
-from share.search.search_params import (
+from share.util.checksum_iri import ChecksumIri
+from trove.trovesearch.search_params import (
     CardsearchParams,
     ValuesearchParams,
 )
-from share.search.search_response import (
+from trove.trovesearch.search_response import (
     CardsearchResponse,
     ValuesearchResponse,
 )
-from share.util.checksum_iri import ChecksumIri
 
 
 logger = logging.getLogger(__name__)

share/search/index_strategy/trove_indexcard_flats.py

@@ -7,7 +7,7 @@
 import logging
 import re
 import uuid
-from typing import Iterable, ClassVar, Optional
+from typing import Iterable, ClassVar, Optional, Iterator
 
 from django.conf import settings
 from django.db.models import Exists, OuterRef
@@ -18,7 +18,9 @@
 from share.search import messages
 from share.search.index_strategy.elastic8 import Elastic8IndexStrategy
 from share.search.index_strategy._util import encode_cursor_dataclass, decode_cursor_dataclass
-from share.search.search_params import (
+from share.util.checksum_iri import ChecksumIri
+from trove import models as trove_db
+from trove.trovesearch.search_params import (
     CardsearchParams,
     ValuesearchParams,
     SearchFilter,
@@ -27,16 +29,14 @@
     PageParam,
     GLOB_PATHSTEP,
 )
-from share.search.search_response import (
+from trove.trovesearch.search_response import (
     CardsearchResponse,
     ValuesearchResponse,
     TextMatchEvidence,
     CardsearchResult,
     ValuesearchResult,
     PropertypathUsage,
 )
-from share.util.checksum_iri import ChecksumIri
-from trove import models as trove_db
 from trove.util.iris import get_sufficiently_unique_iri, is_worthwhile_iri, iri_path_as_keyword
 from trove.vocab.osfmap import is_date_property
 from trove.vocab.namespaces import TROVE, FOAF, RDF, RDFS, DCTERMS, OWL, SKOS, OSFMAP
@@ -689,7 +689,7 @@ def _cardsearch_date_filter(self, search_filter):
                 'query': {'bool': {'filter': list(self._iter_nested_date_filters(search_filter))}},
             }}
 
-        def _iter_nested_date_filters(self, search_filter) -> dict:
+        def _iter_nested_date_filters(self, search_filter) -> Iterator[dict]:
             # filter by requested paths
             yield _pathset_as_nestedvalue_filter(search_filter.propertypath_set, 'nested_date')
             # filter by requested value/operator
@@ -1065,6 +1065,7 @@ def cardsearch_start_index(self) -> int:
 
 class _PredicatePathWalker:
     WalkYield = tuple[tuple[str, ...], primitive_rdf.RdfObject]
+    _visiting: set[str | frozenset]
 
     def __init__(self, tripledict: primitive_rdf.RdfTripleDictionary):
         self.tripledict = tripledict

tests/share/search/index_strategy/_with_real_services.py

@@ -19,8 +19,8 @@ class RealElasticTestCase(TransactionTestCase):
     serialized_rollback = True  # for TransactionTestCase; restore db after
 
     # required for subclasses
-    strategy_name_for_real = None
-    strategy_name_for_test = None
+    strategy_name_for_real: str
+    strategy_name_for_test: str
 
     @classmethod
     def setUpClass(cls):
@@ -38,7 +38,7 @@ def setUp(self):
         )
         self.current_index = self.index_strategy.for_current_index()
         self.current_index.pls_delete()  # in case it already exists
-        self._assert_happypath_until_ingest()
+        self._assert_setup_happypath()
 
     def tearDown(self):
         super().tearDown()
@@ -120,7 +120,7 @@ def _assert_happypath_with_daemon(self, messages_chunk, expected_doc_count):
         else:
             assert False, 'checked and waited but the daemon did not do the thing'
 
-    def _assert_happypath_until_ingest(self):
+    def _assert_setup_happypath(self):
         # initial
         assert not self.current_index.pls_check_exists()
         index_status = self.current_index.pls_get_status()