diff --git a/oarepo_model_builder/datatypes/components/facets/__init__.py b/oarepo_model_builder/datatypes/components/facets/__init__.py index bbb02e7..a9a4a55 100644 --- a/oarepo_model_builder/datatypes/components/facets/__init__.py +++ b/oarepo_model_builder/datatypes/components/facets/__init__.py @@ -14,7 +14,7 @@ class FacetDefinition: dot_path: str searchable: bool imports: List[Dict[str, str]] - facet_groups: List[str] + facet_groups: Dict[str, int] facet: Optional[bool] field: Optional[str] = None diff --git a/oarepo_model_builder/datatypes/components/facets/field.py b/oarepo_model_builder/datatypes/components/facets/field.py index f6d9076..86ab1ac 100644 --- a/oarepo_model_builder/datatypes/components/facets/field.py +++ b/oarepo_model_builder/datatypes/components/facets/field.py @@ -9,6 +9,20 @@ from . import FacetDefinition +class FacetGroupsDict(fields.Dict): + def _deserialize(self, value, attr, data, **kwargs): + transformed_value = {} + if isinstance(value, list): + for val in value: + if isinstance(val, str): + transformed_value[val] = 100000 + else: + transformed_value.update(val) + else: + transformed_value = value + return super()._deserialize(transformed_value, attr, data, **kwargs) + + class FacetsSchema(ma.Schema): class Meta: unknown = ma.RAISE @@ -23,8 +37,9 @@ class Meta: imports = fields.List(fields.Nested(ImportSchema), required=False) path = fields.String(required=False) keyword = fields.String(required=False) - facet_groups = fields.List( - fields.String(), + facet_groups = FacetGroupsDict( + keys=fields.String(), + values=fields.Integer(), required=False, data_key="facet-groups", attribute="facet-groups", @@ -60,7 +75,7 @@ def process_facets(self, datatype, section, **__kwargs): searchable=facet_section.get("searchable"), imports=facet_section.get("imports", []), facet=facet_section.get("facet", None), - facet_groups=facet_section.get("facet-groups", ["_default"]), + facet_groups=facet_section.get("facet-groups", {"_default": 100000}), ) # set the field on the definition diff --git a/oarepo_model_builder/datatypes/components/model/facets.py b/oarepo_model_builder/datatypes/components/model/facets.py index c5dda28..2a6e07a 100644 --- a/oarepo_model_builder/datatypes/components/model/facets.py +++ b/oarepo_model_builder/datatypes/components/model/facets.py @@ -24,6 +24,16 @@ class Meta: ) generate = ma.fields.Boolean() skip = ma.fields.Boolean() + facet_groups = ma.fields.Dict( + attribute="facet-groups", + data_key="facet-groups", + keys=ma.fields.String(), + values=ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.Integer()), + metadata={ + "doc": "Groups of facets in the form of {group_name: {facet_path: priority}}. Will merge with facet " + "groups declared on the individual fields." + }, + ) class FacetsModelComponent(ObjectFacetsComponent): @@ -45,6 +55,7 @@ def before_model_prepare(self, datatype, *, context, **__kwargs): facets.setdefault("module", f"{module}.services.{profile_module}.facets") facets.setdefault("extra-code", "") + facets.setdefault("groups", True) def build_facet_definition( self, diff --git a/oarepo_model_builder/datatypes/components/model/record.py b/oarepo_model_builder/datatypes/components/model/record.py index 00ea3db..8796ae4 100644 --- a/oarepo_model_builder/datatypes/components/model/record.py +++ b/oarepo_model_builder/datatypes/components/model/record.py @@ -36,6 +36,11 @@ class Meta: ma.fields.Nested(ImportSchema), metadata={"doc": "List of python imports"} ) skip = ma.fields.Boolean() + fields = ma.fields.Dict( + attribute="fields", + data_key="fields", + metadata={"doc": "Extra fields to add to the class"}, + ) class RecordModelComponent(DataTypeComponent): @@ -65,4 +70,5 @@ def before_model_prepare(self, datatype, *, context, **kwargs): [], ) record.setdefault("extra-code", "") + record.setdefault("fields", {}) convert_config_to_qualified_name(record) diff --git a/oarepo_model_builder/datatypes/components/model/search_options.py b/oarepo_model_builder/datatypes/components/model/search_options.py index 71bf825..7ce98c5 100644 --- a/oarepo_model_builder/datatypes/components/model/search_options.py +++ b/oarepo_model_builder/datatypes/components/model/search_options.py @@ -41,6 +41,12 @@ class Meta: data_key="sort-options-field", ) + fields = ma.fields.Dict( + keys=ma.fields.Str(), + values=ma.fields.Str(), + metadata={"doc": "Fields to be used in search options"}, + ) + class SearchOptionsModelComponent(DataTypeComponent): eligible_datatypes = [ModelDataType] @@ -77,4 +83,8 @@ def before_model_prepare(self, datatype, *, context, **kwargs): "imports", [], ) + record_search_options.setdefault( + "fields", + {}, + ) record_search_options.setdefault("sort-options-field", "sort_options") diff --git a/oarepo_model_builder/invenio/invenio_record_search_options.py b/oarepo_model_builder/invenio/invenio_record_search_options.py index f3a26f4..e9b6134 100644 --- a/oarepo_model_builder/invenio/invenio_record_search_options.py +++ b/oarepo_model_builder/invenio/invenio_record_search_options.py @@ -1,3 +1,4 @@ +from collections import defaultdict from typing import List from ..datatypes.components.facets import FacetDefinition @@ -15,19 +16,62 @@ def finish(self, **extra_kwargs): facet_groups = {} default_group = [] search_data = [] + + # gather all facet group names + facet_group_names = set() + for f in facets: + facet_group_names.update(f.facet_groups.keys()) + + # there might be a definition of facet groups on the model's facets -> facet-groups + # if there is, we need to merge those + top_level_facet_groups = self.current_model.definition.get("facets", {}).get( + "facet-groups", {} + ) + facet_group_names.update(top_level_facet_groups.keys()) + path_to_facet_group = defaultdict(dict) + + for group, group_def in top_level_facet_groups.items(): + for path, priority in group_def.items(): + path_to_facet_group[group][path] = priority + + # for each group name, gather all facets with that group name + # sort them by their order in the group + # and add them to the facet_groups dict + for group in sorted(facet_group_names): + # skip the default group + if group == "_default": + continue + + # gather all facets with this facet_group_name + group_members = [] + for f in facets: + if group in f.facet_groups: + group_members.append((f, f.facet_groups[group])) + + # if there is a definition of facet groups on the model's facets -> add the selected facets as well + for path, priority in path_to_facet_group.get(group, {}).items(): + if f.dot_path.startswith(path): + group_members.append((f, priority)) + + # sort the group members by their order in the facet_group + group_members.sort(key=lambda x: x[1]) + + if group not in facet_groups.keys(): + facet_groups[group] = {} + + for f, _ in group_members: + facet_groups[group][f.path] = "facets." + f.path + for f in facets: - for group in f.facet_groups: - if group != "_default": - if group not in facet_groups.keys(): - facet_groups[group] = {} - facet_groups[group][f.path] = "facets." + f.path if len(f.facet_groups) > 0: default_group.append({f.path: "facets." + f.path}) search_data.append({f.path: "facets." + f.path}) + if "sortable" in self.current_model.definition: sort_options = self.current_model.definition["sortable"] else: sort_options = {} + extra_kwargs["facet_groups"] = facet_groups extra_kwargs["default_group"] = default_group extra_kwargs["sort_definition"] = sort_options diff --git a/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 b/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 index 4e30fcc..feddd98 100644 --- a/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 +++ b/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 @@ -3,6 +3,10 @@ from oarepo_runtime.i18n import lazy_gettext as _ from . import facets +{% for extra_field_value in vars.search_options.fields.values() %} +{{ extra_field_value|code_imports }} +{% endfor %} + class {{ vars.search_options| class_header }}: """{{ vars.record.class|base_name }} search options.""" @@ -49,5 +53,8 @@ class {{ vars.search_options| class_header }}: {% endfor %} } {% endif %} +{% for extra_field_name, extra_field_value in vars.search_options.fields.items() %} + {{ extra_field_name }} = {{ extra_field_value|extra_code }} +{% endfor %} {{ vars.search_options|extra_code }} \ No newline at end of file diff --git a/oarepo_model_builder/invenio/templates/record.py.jinja2 b/oarepo_model_builder/invenio/templates/record.py.jinja2 index 74fc8a0..1a8433b 100644 --- a/oarepo_model_builder/invenio/templates/record.py.jinja2 +++ b/oarepo_model_builder/invenio/templates/record.py.jinja2 @@ -9,6 +9,11 @@ from invenio_records_resources.records.systemfields import IndexField {{ vars.record_dumper.class|imports }} {{ vars.record|imports }} +{% for extra_field_value in vars.record.fields.values() %} +{{ extra_field_value|code_imports }} +{% endfor %} + + class {{ vars.record|class_header }}: {% if not vars.record_metadata.skip %} model_cls = {{ vars.record_metadata.class|base_name }} @@ -30,5 +35,8 @@ class {{ vars.record|class_header }}: {% if not vars.record_dumper.skip %} dumper = {{ vars.record_dumper.class|base_name }}() {% endif %} +{% for extra_field_name, extra_field_value in vars.record.fields.items() %} + {{ extra_field_name }} = {{ extra_field_value|extra_code }} +{% endfor %} {{ vars.record|extra_code }} \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 7f552d5..a1075c6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = oarepo-model-builder -version = 4.0.74 +version = 4.0.75 description = A utility library that generates OARepo required data model files from a JSON specification file authors = Miroslav Bauer , Miroslav Simek readme = README.md diff --git a/tests/test_datatype_prepare.py b/tests/test_datatype_prepare.py index 8a0138f..9642478 100644 --- a/tests/test_datatype_prepare.py +++ b/tests/test_datatype_prepare.py @@ -76,6 +76,7 @@ def test_prepare_datatype(): "invenio_records_resources.services.SearchOptions{InvenioSearchOptions}" ], "imports": [], + "fields": {}, "sort-options-field": "sort_options", }, "config": { @@ -112,6 +113,7 @@ def test_prepare_datatype(): "facets": { "generate": True, "module": "my.test.services.records.facets", + 'groups': True, "extra-code": "", }, "record": { @@ -123,6 +125,7 @@ def test_prepare_datatype(): ], "imports": [], "extra-code": "", + "fields": {}, }, "resource": { "generate": True, diff --git a/tests/test_model_saver.py b/tests/test_model_saver.py index c2a8712..bb5f3b3 100644 --- a/tests/test_model_saver.py +++ b/tests/test_model_saver.py @@ -70,6 +70,7 @@ def test_model_saver(): "invenio_records_resources.services.SearchOptions{InvenioSearchOptions}" ], "imports": [], + "fields": {}, "sort-options-field": "sort_options", }, "config": { @@ -106,6 +107,7 @@ def test_model_saver(): "facets": { "generate": True, "module": "test.services.records.facets", + 'groups': True, "extra-code": "", }, "record": { @@ -117,6 +119,7 @@ def test_model_saver(): ], "imports": [], "extra-code": "", + "fields": {}, }, "resource": { "generate": True, @@ -438,6 +441,7 @@ def test_model_saver_invenio(): "invenio_records_resources.services.SearchOptions{InvenioSearchOptions}" ], "imports": [], + "fields": {}, "sort-options-field": "sort_options", }, "config": { @@ -474,6 +478,7 @@ def test_model_saver_invenio(): "facets": { "generate": True, "module": "test.services.records.facets", + "groups": True, "extra-code": "", }, "record": { @@ -485,6 +490,7 @@ def test_model_saver_invenio(): ], "imports": [], "extra-code": "", + "fields": {}, }, "resource": { "generate": True, diff --git a/tests/test_search_options.py b/tests/test_search_options.py index 3fd28db..2b8e868 100644 --- a/tests/test_search_options.py +++ b/tests/test_search_options.py @@ -316,40 +316,19 @@ class TestSearchOptions(BaseSearchOptions): \"""TestRecord search options.\""" facet_groups ={ - 'curator': { - 'a' : facets.a, - - 'g' : facets.g, - - - **getattr(BaseSearchOptions, 'facet_groups', {}).get('curator', {}) - }, - - 'user': { - - 'a' : facets.a, - - - - **getattr(BaseSearchOptions, 'facet_groups', {}).get('user', {}) - - }, - 'default': { - 'b3' : facets.b3, - - - **getattr(BaseSearchOptions, 'facet_groups', {}).get('default', {}) - }, - + 'user': { + 'a' : facets.a, + **getattr(BaseSearchOptions, 'facet_groups', {}).get('user', {}) + }, } facets = { @@ -382,6 +361,140 @@ class TestSearchOptions(BaseSearchOptions): ) +def test_facet_groups_ordering(): + schema = load_model( + DUMMY_YAML, + model_content={ + "record": { + "use": "invenio", + "module": {"qualified": "test"}, + "search-options": { + "base-classes": ["BaseSearchOptions"], + "imports": [{"import": "blah.BaseSearchOptions"}], + }, + "properties": { + "a": { + "type": "keyword", + "facets": {"facet-groups": {"curator": 2}}, + }, + "b": { + "type": "keyword", + "facets": {"facet-groups": {"curator": 3}}, + }, + "c": { + "type": "keyword", + "facets": {"facet-groups": {"curator": 1}}, + }, + }, + }, + }, + isort=False, + black=False, + autoflake=False, + ) + + filesystem = InMemoryFileSystem() + builder = create_builder_from_entrypoints(filesystem=filesystem) + + builder.build(schema, "record", ["record"], "") + + data = builder.filesystem.open( + os.path.join("test", "services", "records", "search.py") + ).read() + print(data) + data2 = builder.filesystem.open( + os.path.join("test", "services", "records", "facets.py") + ).read() + print(data2) + data3 = builder.filesystem.read( + os.path.join("test", "records", "mappings", "os-v2", "test", "test-1.0.0.json") + ) + import json + + data3 = json.loads(data3) + print(data3) + assert ( + strip_whitespaces( + """ + 'c': facets.c, + 'a': facets.a, + 'b': facets.b, + """ + ) + in strip_whitespaces(data) + ) + + +def test_top_level_facet_groups_ordering(): + schema = load_model( + DUMMY_YAML, + model_content={ + "record": { + "use": "invenio", + "module": {"qualified": "test"}, + "search-options": { + "base-classes": ["BaseSearchOptions"], + "imports": [{"import": "blah.BaseSearchOptions"}], + }, + "facets": { + "facet-groups": { + "curator": { + "a": 2, + "b": 3, + "c": 1, + } + } + }, + "properties": { + "a": { + "type": "keyword", + }, + "b": { + "type": "keyword", + }, + "c": { + "type": "keyword", + }, + }, + }, + }, + isort=False, + black=False, + autoflake=False, + ) + + filesystem = InMemoryFileSystem() + builder = create_builder_from_entrypoints(filesystem=filesystem) + + builder.build(schema, "record", ["record"], "") + + data = builder.filesystem.open( + os.path.join("test", "services", "records", "search.py") + ).read() + print(data) + data2 = builder.filesystem.open( + os.path.join("test", "services", "records", "facets.py") + ).read() + print(data2) + data3 = builder.filesystem.read( + os.path.join("test", "records", "mappings", "os-v2", "test", "test-1.0.0.json") + ) + import json + + data3 = json.loads(data3) + print(data3) + assert ( + strip_whitespaces( + """ + 'c': facets.c, + 'a': facets.a, + 'b': facets.b, + """ + ) + in strip_whitespaces(data) + ) + + def test_replace_sort_options(): schema = load_model( DUMMY_YAML,