diff --git a/src/onegov/activity/models/attendee.py b/src/onegov/activity/models/attendee.py index efc3a943d2..1538a8c7ec 100644 --- a/src/onegov/activity/models/attendee.py +++ b/src/onegov/activity/models/attendee.py @@ -1,4 +1,5 @@ from datetime import date + from onegov.activity.models.booking import Booking from onegov.core.orm import Base from onegov.core.orm.mixins import TimestampMixin @@ -11,7 +12,6 @@ from sqlalchemy import Date from sqlalchemy import Float from sqlalchemy import ForeignKey -from sqlalchemy import Index from sqlalchemy import Integer from sqlalchemy import Numeric from sqlalchemy import Text @@ -43,6 +43,10 @@ class Attendee(Base, TimestampMixin, ORMSearchable): } es_public = False + @property + def search_score(self): + return 3 + @property def es_suggestion(self): return self.name @@ -179,7 +183,3 @@ def happiness(cls, period_id): order_by='Booking.created', backref='attendee' ) - - __table_args__ = ( - Index('unique_child_name', 'username', 'name', unique=True), - ) diff --git a/src/onegov/agency/models/person.py b/src/onegov/agency/models/person.py index 6a62a2499b..2a56f24f8c 100644 --- a/src/onegov/agency/models/person.py +++ b/src/onegov/agency/models/person.py @@ -21,8 +21,6 @@ def es_public(self): 'title': {'type': 'text'}, 'function': {'type': 'localized'}, 'email': {'type': 'text'}, - 'phone_internal': {'type': 'text'}, - 'phone_es': {'type': 'text'} } @property diff --git a/src/onegov/agency/views/search.py b/src/onegov/agency/views/search.py index ff197aaaf1..740159649b 100644 --- a/src/onegov/agency/views/search.py +++ b/src/onegov/agency/views/search.py @@ -1,13 +1,23 @@ from onegov.agency import AgencyApp from onegov.agency.layout import AgencySearchLayout from onegov.core.security import Public -from onegov.org.models import Search +from onegov.org.models import Search, SearchPostgres from onegov.org.views.search import search as search_view +from onegov.org.views.search import search_postgres as search_postgres_view @AgencyApp.html(model=Search, template='search.pt', permission=Public) -def search(self, request): +def agency_search(self, request): data = search_view(self, request) if isinstance(data, dict): data['layout'] = AgencySearchLayout(self, request) return data + + +@AgencyApp.html(model=SearchPostgres, template='search_postgres.pt', + permission=Public) +def agency_search_postgres(self, request): + data = search_postgres_view(self, request) + if isinstance(data, dict): + data['layout'] = AgencySearchLayout(self, request) + return data diff --git a/src/onegov/core/upgrade.py b/src/onegov/core/upgrade.py index 1b409fafdc..eff83c4ebe 100644 --- a/src/onegov/core/upgrade.py +++ b/src/onegov/core/upgrade.py @@ -480,6 +480,12 @@ def has_column(self, table: str, column: str) -> bool: table, schema=self.schema )} + def has_index(self, table: str, index: str) -> bool: + inspector = Inspector(self.operations_connection) + return index in {i['name'] for i in inspector.get_indexes( + table, schema=self.schema + )} + def has_enum(self, enum: str) -> bool: return self.session.execute(f""" SELECT EXISTS ( diff --git a/src/onegov/directory/models/directory.py b/src/onegov/directory/models/directory.py index 6c1c66d0c3..c03f3a224f 100644 --- a/src/onegov/directory/models/directory.py +++ b/src/onegov/directory/models/directory.py @@ -17,10 +17,8 @@ from onegov.file.utils import as_fileintent from onegov.form import flatten_fieldsets, parse_formcode, parse_form from onegov.search import SearchableContent -from sqlalchemy import Column +from sqlalchemy import Column, Text, Integer from sqlalchemy import func, exists, and_ -from sqlalchemy import Integer -from sqlalchemy import Text from sqlalchemy.orm import object_session from sqlalchemy.orm import relationship from sqlalchemy.orm.attributes import InstrumentedAttribute @@ -103,6 +101,10 @@ def count(self): backref='directory' ) + @property + def search_score(self): + return 7 + @property def entry_cls_name(self): return 'DirectoryEntry' diff --git a/src/onegov/directory/models/directory_entry.py b/src/onegov/directory/models/directory_entry.py index 4f92fe3828..49d0fb9ce2 100644 --- a/src/onegov/directory/models/directory_entry.py +++ b/src/onegov/directory/models/directory_entry.py @@ -6,12 +6,13 @@ from onegov.file import AssociatedFiles from onegov.gis import CoordinatesMixin from onegov.search import SearchableContent -from sqlalchemy import Column +from sqlalchemy import Column, cast from sqlalchemy import ForeignKey from sqlalchemy import Index from sqlalchemy import Text from sqlalchemy.dialects.postgresql import HSTORE from sqlalchemy.ext.mutable import MutableDict +from sqlalchemy.ext.hybrid import hybrid_property from uuid import uuid4 @@ -25,7 +26,7 @@ class DirectoryEntry(Base, ContentMixin, CoordinatesMixin, TimestampMixin, 'keywords': {'type': 'keyword'}, 'title': {'type': 'localized'}, 'lead': {'type': 'localized'}, - 'directory_id': {'type': 'keyword'}, + '_directory_id': {'type': 'keyword'}, # since the searchable text might include html, we remove it # even if there's no html -> possibly decreasing the search @@ -86,6 +87,10 @@ def external_link_title(self): def external_link_visible(self): return self.directory.configuration.link_visible + @hybrid_property + def _directory_id(self): + return cast(self.directory_id, Text) + @property def directory_name(self): return self.directory.name @@ -98,7 +103,7 @@ def keywords(self): def keywords(self, value): self._keywords = {k: '' for k in value} if value else None - @property + @hybrid_property def text(self): return self.directory.configuration.extract_searchable(self.values) diff --git a/src/onegov/event/models/event.py b/src/onegov/event/models/event.py index f69e9ef089..6d4107050a 100644 --- a/src/onegov/event/models/event.py +++ b/src/onegov/event/models/event.py @@ -6,6 +6,7 @@ from icalendar import Calendar as vCalendar from icalendar import Event as vEvent from icalendar import vRecur +from sqlalchemy.ext.hybrid import hybrid_property from onegov.core.orm import Base from onegov.core.orm.abstract import associated @@ -109,6 +110,10 @@ class Event(Base, OccurrenceMixin, TimestampMixin, SearchableContent, EventFile, 'pdf', 'one-to-one', uselist=False, backref_suffix='pdf' ) + @property + def search_score(self): + return 1 + def set_image(self, content, filename=None): self.set_blob('image', content, filename) @@ -148,9 +153,16 @@ def set_blob(self, blob, content, filename=None): 'description': {'type': 'localized'}, 'location': {'type': 'localized'}, 'organizer': {'type': 'localized'}, - 'filter_keywords': {'type': 'keyword'} } + @hybrid_property + def description(self): # noqa: F811 + return self.content['description'].astext + + @hybrid_property + def organizer(self): # noqa: F811 + return self.content['organizer'].astext + @property def es_public(self): return self.state == 'published' diff --git a/src/onegov/feriennet/models/activity.py b/src/onegov/feriennet/models/activity.py index 4665613c14..42bc2e6bcd 100644 --- a/src/onegov/feriennet/models/activity.py +++ b/src/onegov/feriennet/models/activity.py @@ -1,4 +1,7 @@ from functools import cached_property + +from sqlalchemy.ext.hybrid import hybrid_property + from onegov.activity import Activity, ActivityCollection, Occasion from onegov.activity import PublicationRequestCollection from onegov.activity.models import DAYS @@ -24,6 +27,10 @@ class VacationActivity(Activity, CoordinatesExtension, SearchableContent): 'organiser': {'type': 'text'} } + @property + def search_score(self): + return 1 + @property def es_public(self): return self.state == 'accepted' @@ -32,7 +39,7 @@ def es_public(self): def es_skip(self): return self.state == 'preview' - @property + @hybrid_property def organiser(self): organiser = [ self.user.username, diff --git a/src/onegov/feriennet/templates/mail_booking_accepted.pt b/src/onegov/feriennet/templates/mail_booking_accepted.pt index 46ada5cd6e..ebd8c3c1c2 100644 --- a/src/onegov/feriennet/templates/mail_booking_accepted.pt +++ b/src/onegov/feriennet/templates/mail_booking_accepted.pt @@ -16,7 +16,7 @@


-

Best regards

+

Best regards

diff --git a/src/onegov/feriennet/templates/mail_booking_canceled.pt b/src/onegov/feriennet/templates/mail_booking_canceled.pt index 5b272ec3fe..1e35c9543a 100644 --- a/src/onegov/feriennet/templates/mail_booking_canceled.pt +++ b/src/onegov/feriennet/templates/mail_booking_canceled.pt @@ -15,7 +15,7 @@


-

Best regards

+

Best regards

diff --git a/src/onegov/file/models/file.py b/src/onegov/file/models/file.py index 6806ce330f..3ce25ca429 100644 --- a/src/onegov/file/models/file.py +++ b/src/onegov/file/models/file.py @@ -6,6 +6,7 @@ from contextlib import contextmanager from collections import defaultdict from depot.fields.sqlalchemy import UploadedFileField as UploadedFileFieldBase + from onegov.core.crypto import random_token from onegov.core.orm import Base from onegov.core.orm.abstract import Associable @@ -262,6 +263,10 @@ class File(Base, Associable, TimestampMixin): Index('files_by_type_and_order', 'type', 'order'), ) + @property + def search_score(self) -> int: + return 10 + @hybrid_property def signature_timestamp(self) -> 'datetime | None': if self.signed: diff --git a/src/onegov/file/models/fileset.py b/src/onegov/file/models/fileset.py index 5ae1a1c37e..51cccbbd2c 100644 --- a/src/onegov/file/models/fileset.py +++ b/src/onegov/file/models/fileset.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING + if TYPE_CHECKING: from .file import File diff --git a/src/onegov/form/models/definition.py b/src/onegov/form/models/definition.py index 26426a04cb..85673fb2c5 100644 --- a/src/onegov/form/models/definition.py +++ b/src/onegov/form/models/definition.py @@ -140,6 +140,9 @@ class FormDefinition(Base, ContentMixin, TimestampMixin, Extendable): } @property + def search_score(self) -> int: + return 7 + def form_class(self) -> Type['Form']: """ Parses the form definition and returns a form class. """ diff --git a/src/onegov/fsi/models/course.py b/src/onegov/fsi/models/course.py index 5c79d353da..04c6957469 100644 --- a/src/onegov/fsi/models/course.py +++ b/src/onegov/fsi/models/course.py @@ -1,4 +1,5 @@ from arrow import utcnow + from onegov.core.html import html_to_text from onegov.core.orm import Base from onegov.core.orm.types import UUID @@ -32,6 +33,10 @@ class Course(Base, ORMSearchable): # hides the course in the collection for non-admins hidden_from_public = Column(Boolean, nullable=False, default=False) + @property + def search_score(self): + return 2 + @property def title(self): return self.name diff --git a/src/onegov/fsi/models/course_attendee.py b/src/onegov/fsi/models/course_attendee.py index 5a243ed852..ceaee8127c 100644 --- a/src/onegov/fsi/models/course_attendee.py +++ b/src/onegov/fsi/models/course_attendee.py @@ -1,3 +1,5 @@ +from sqlalchemy.ext.hybrid import hybrid_property + from onegov.core.orm import Base from onegov.core.orm.types import UUID, JSON from sqlalchemy import Boolean @@ -107,7 +109,7 @@ def __str__(self): cascade='all, delete-orphan' ) - @property + @hybrid_property def title(self): return ' '.join(( p for p in ( @@ -131,7 +133,7 @@ def role(self): return 'member' return self.user.role - @property + @hybrid_property def email(self): """Needs a switch for external users""" if not self.user_id: diff --git a/src/onegov/fsi/models/course_event.py b/src/onegov/fsi/models/course_event.py index cbe27dd4d4..706302a419 100644 --- a/src/onegov/fsi/models/course_event.py +++ b/src/onegov/fsi/models/course_event.py @@ -22,13 +22,15 @@ from onegov.fsi.models.course_subscription import subscription_table from onegov.search import ORMSearchable - from typing import TYPE_CHECKING if TYPE_CHECKING: from .course import Course from .course_notification_template import ( - CancellationTemplate, CourseNotificationTemplate, InfoTemplate, - ReminderTemplate, SubscriptionTemplate + CancellationTemplate, + CourseNotificationTemplate, + InfoTemplate, + ReminderTemplate, + SubscriptionTemplate ) COURSE_EVENT_STATUSES = ('created', 'confirmed', 'canceled', 'planned') @@ -83,7 +85,7 @@ def es_public(self): def title(self): return str(self) - @property + @hybrid_property def name(self): return self.course.name @@ -95,7 +97,7 @@ def lead(self): f'{self.presenter_company}' ) - @property + @hybrid_property def description(self): return self.course.description diff --git a/src/onegov/fsi/views/search.py b/src/onegov/fsi/views/search.py index 22a659a330..b1b3c8980f 100644 --- a/src/onegov/fsi/views/search.py +++ b/src/onegov/fsi/views/search.py @@ -1,8 +1,11 @@ from onegov.core.security import Personal from onegov.fsi import FsiApp -from onegov.org.models import Search +from onegov.org.models import Search, SearchPostgres from onegov.org.views.search import search as search_view +from onegov.org.views.search import search_postgres as search_postgres_view from onegov.org.views.search import suggestions as suggestions_view +from onegov.org.views.search import suggestions_postgres as \ + suggestions_postgres_view @FsiApp.html(model=Search, template='search.pt', permission=Personal) @@ -10,6 +13,17 @@ def search(self, request): return search_view(self, request) +@FsiApp.html(model=SearchPostgres, template='search_postgres.pt', + permission=Personal) +def search_postgres(self, request): + return search_postgres_view(self, request) + + @FsiApp.json(model=Search, name='suggest', permission=Personal) def suggestions(self, request): return suggestions_view(self, request) + + +@FsiApp.json(model=SearchPostgres, name='suggest', permission=Personal) +def suggestions_postgres(self, request): + return suggestions_postgres_view(self, request) diff --git a/src/onegov/landsgemeinde/models/agenda.py b/src/onegov/landsgemeinde/models/agenda.py index e75e018dad..4f6f26f123 100644 --- a/src/onegov/landsgemeinde/models/agenda.py +++ b/src/onegov/landsgemeinde/models/agenda.py @@ -1,3 +1,5 @@ +from sqlalchemy.ext.hybrid import hybrid_property + from onegov.core.orm import Base from onegov.core.orm.mixins import content_property from onegov.core.orm.mixins import ContentMixin @@ -105,6 +107,18 @@ class AgendaItem( last_modified = Column(UTCDateTime) + @hybrid_property + def overview(self): # noqa: F811 + return self.content['overview'].astext + + @hybrid_property + def text(self): # noqa: F811 + return self.content['text'].astext + + @hybrid_property + def resolution(self): # noqa: F811 + return self.content['resolution'].astext + def stamp(self): self.last_modified = self.timestamp() diff --git a/src/onegov/landsgemeinde/models/assembly.py b/src/onegov/landsgemeinde/models/assembly.py index ed4d4367a1..e4124be56d 100644 --- a/src/onegov/landsgemeinde/models/assembly.py +++ b/src/onegov/landsgemeinde/models/assembly.py @@ -1,3 +1,5 @@ +from sqlalchemy.ext.hybrid import hybrid_property + from onegov.core.orm import Base from onegov.core.orm.mixins import content_property from onegov.core.orm.mixins import ContentMixin @@ -94,5 +96,9 @@ def es_suggestion(self): last_modified = Column(UTCDateTime) + @hybrid_property + def overview(self): # noqa: F811 + return self.content['overview'].astext + def stamp(self): self.last_modified = self.timestamp() diff --git a/src/onegov/landsgemeinde/models/votum.py b/src/onegov/landsgemeinde/models/votum.py index 96ab78e552..b5c3746de2 100644 --- a/src/onegov/landsgemeinde/models/votum.py +++ b/src/onegov/landsgemeinde/models/votum.py @@ -1,3 +1,5 @@ +from sqlalchemy.ext.hybrid import hybrid_property + from onegov.core.orm import Base from onegov.core.orm.mixins import content_property from onegov.core.orm.mixins import ContentMixin @@ -92,6 +94,18 @@ def es_suggestion(self): nullable=False ) + @hybrid_property + def text(self): # noqa: F811 + return self.content['text'].astext + + @hybrid_property + def motion(self): # noqa: F811 + return self.content['motion'].astext + + @hybrid_property + def statement_of_reasons(self): # noqa: F811 + return self.content['statement_of_reasons'].astext + @property def date(self): return self.agenda_item.date diff --git a/src/onegov/landsgemeinde/views/search.py b/src/onegov/landsgemeinde/views/search.py index 04d3e17b1a..ef759e8166 100644 --- a/src/onegov/landsgemeinde/views/search.py +++ b/src/onegov/landsgemeinde/views/search.py @@ -7,4 +7,5 @@ @LandsgemeindeApp.html(model=Search, template='search.pt', permission=Public) def landsgemeinde_search(self, request): + # TODO: switch to postgres search return search(self, request, DefaultLayout(self, request)) diff --git a/src/onegov/newsletter/models.py b/src/onegov/newsletter/models.py index 44500057d0..9e93a7a87b 100644 --- a/src/onegov/newsletter/models.py +++ b/src/onegov/newsletter/models.py @@ -95,6 +95,9 @@ def validate_name(self, key: str, name: str) -> str: back_populates='newsletters') @property + def search_score(self) -> int: + return 6 + def open_recipients(self) -> tuple['Recipient', ...]: received = select([newsletter_recipients.c.recipient_id]).where( newsletter_recipients.c.newsletter_id == self.name) diff --git a/src/onegov/onboarding/app.py b/src/onegov/onboarding/app.py index a460033181..ab5864d9b4 100644 --- a/src/onegov/onboarding/app.py +++ b/src/onegov/onboarding/app.py @@ -2,7 +2,7 @@ from onegov.file import DepotApp from onegov.onboarding.theme import OnboardingTheme from onegov.reservation import LibresIntegration -from onegov.search import ElasticsearchApp +from onegov.search import SearchApp from typing import Any, TYPE_CHECKING @@ -10,7 +10,7 @@ from collections.abc import Iterator -class OnboardingApp(Framework, LibresIntegration, DepotApp, ElasticsearchApp): +class OnboardingApp(Framework, LibresIntegration, DepotApp, SearchApp): serve_static_files = True diff --git a/src/onegov/org/app.py b/src/onegov/org/app.py index 120b5d71e4..450c10efb2 100644 --- a/src/onegov/org/app.py +++ b/src/onegov/org/app.py @@ -29,7 +29,7 @@ from onegov.page import Page, PageCollection from onegov.pay import PayApp from onegov.reservation import LibresIntegration -from onegov.search import ElasticsearchApp +from onegov.search import SearchApp from onegov.ticket import TicketCollection from onegov.ticket import TicketPermission from onegov.user import UserApp @@ -47,7 +47,7 @@ from reg.dispatch import _KeyLookup -class OrgApp(Framework, LibresIntegration, ElasticsearchApp, MapboxApp, +class OrgApp(Framework, LibresIntegration, SearchApp, MapboxApp, DepotApp, PayApp, FormApp, UserApp, WebsocketsApp): serve_static_files = True diff --git a/src/onegov/org/layout.py b/src/onegov/org/layout.py index 9943c2cd9a..586a2290d3 100644 --- a/src/onegov/org/layout.py +++ b/src/onegov/org/layout.py @@ -26,7 +26,7 @@ from onegov.org import _ from onegov.org import utils from onegov.org.exports.base import OrgExport -from onegov.org.models import ExportCollection, Editor +from onegov.org.models import ExportCollection, Editor, SearchPostgres from onegov.org.models import GeneralFileCollection from onegov.org.models import ImageFile from onegov.org.models import ImageFileCollection @@ -290,11 +290,16 @@ def homepage_url(self): @cached_property def search_url(self): """ Returns the url to the search page. """ + if 'search_postgres' in self.request.path_info: + return self.request.link(SearchPostgres(self.request, None, None)) return self.request.link(Search(self.request, None, None)) @cached_property def suggestions_url(self): """ Returns the url to the suggestions json view. """ + if 'search_postgres' in self.request.path_info: + return self.request.link(SearchPostgres(self.request, None, + None), 'suggest') return self.request.link(Search(self.request, None, None), 'suggest') @cached_property diff --git a/src/onegov/org/models/__init__.py b/src/onegov/org/models/__init__.py index 32ac6f3b55..a5102c6073 100644 --- a/src/onegov/org/models/__init__.py +++ b/src/onegov/org/models/__init__.py @@ -46,7 +46,7 @@ from onegov.org.models.recipient import ResourceRecipient from onegov.org.models.recipient import ResourceRecipientCollection from onegov.org.models.resource import DaypassResource -from onegov.org.models.search import Search +from onegov.org.models.search import Search, SearchPostgres from onegov.org.models.sitecollection import SiteCollection from onegov.org.models.swiss_holidays import SwissHolidays from onegov.org.models.tan import TAN @@ -103,6 +103,7 @@ 'ResourceRecipient', 'ResourceRecipientCollection', 'Search', + 'SearchPostgres', 'SiteCollection', 'SubmissionMessage', 'SwissHolidays', diff --git a/src/onegov/org/models/external_link.py b/src/onegov/org/models/external_link.py index 600e1a5276..f3f06a428b 100644 --- a/src/onegov/org/models/external_link.py +++ b/src/onegov/org/models/external_link.py @@ -1,5 +1,7 @@ from uuid import uuid4 +from sqlalchemy.ext.hybrid import hybrid_property + from onegov.core.collection import GenericCollection from onegov.core.orm import Base from onegov.core.orm.mixins import ContentMixin, \ @@ -48,10 +50,18 @@ class ExternalLink(Base, ContentMixin, TimestampMixin, AccessExtension, lead = meta_property() + @property + def search_score(self): + return 8 + @observes('title') def title_observer(self, title): self.order = normalize_for_url(title) + @hybrid_property + def lead(self): # noqa: F811 + return self.meta['lead'].astext + class ExternalLinkCollection(GenericCollection): diff --git a/src/onegov/org/models/file.py b/src/onegov/org/models/file.py index 49f2ef3d65..dbe729863e 100644 --- a/src/onegov/org/models/file.py +++ b/src/onegov/org/models/file.py @@ -7,6 +7,9 @@ from dateutil.relativedelta import relativedelta from functools import cached_property from itertools import chain, groupby + +from sqlalchemy.ext.hybrid import hybrid_property + from onegov.core.orm import as_selectable from onegov.core.orm.mixins import meta_property from onegov.file import File, FileSet, FileCollection, FileSetCollection @@ -153,6 +156,14 @@ def es_suggestions(self): show_images_on_homepage = meta_property() + @hybrid_property + def lead(self): # noqa: F811 + return self.meta['lead'].astext + + @hybrid_property + def view(self): # noqa: F811 + return self.meta['view'].astext + class ImageSetCollection(FileSetCollection): diff --git a/src/onegov/org/models/form.py b/src/onegov/org/models/form.py index 24c3a67e3a..31aac78788 100644 --- a/src/onegov/org/models/form.py +++ b/src/onegov/org/models/form.py @@ -1,3 +1,5 @@ +from sqlalchemy.ext.hybrid import hybrid_property + from onegov.form.models import FormDefinition from onegov.org.models.extensions import AccessExtension from onegov.org.models.extensions import ContactExtension @@ -21,6 +23,14 @@ class BuiltinFormDefinition(FormDefinition, AccessExtension, def extensions(self): return tuple(set(super().extensions + ['honeypot'])) + @hybrid_property + def lead(self): + return self.meta['lead'].astext + + @hybrid_property + def text(self): + return self.content['text'].astext + class CustomFormDefinition(FormDefinition, AccessExtension, ContactExtension, PersonLinkExtension, diff --git a/src/onegov/org/models/page.py b/src/onegov/org/models/page.py index 4d7ab1e90a..05b9dfbcab 100644 --- a/src/onegov/org/models/page.py +++ b/src/onegov/org/models/page.py @@ -1,4 +1,5 @@ from datetime import datetime + from onegov.core.orm.mixins import ( content_property, dict_property, meta_property) from onegov.file import MultiAssociatedFiles @@ -21,6 +22,7 @@ from sedate import replace_timezone from sqlalchemy import desc, func, or_, and_ from sqlalchemy.dialects.postgresql import array, JSON +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import undefer, object_session from sqlalchemy_utils import observes @@ -42,6 +44,30 @@ class Topic(Page, TraitInfo, SearchableContent, AccessExtension, # Show the lead on topics page lead_when_child = content_property(default=True) + @hybrid_property + def lead(self): # noqa: F811 + return self.content['lead'] + + @lead.expression + def lead(cls): + return cls.content['lead'].astext + + @hybrid_property + def text(self): # noqa: F811 + return self.content['text'] + + @text.expression + def text(cls): + return cls.content['text'].astext + + @hybrid_property + def url(self): # noqa: F811 + return self.content['url'] + + @url.expression + def url(cls): + return cls.content['url'].astext + @property def es_skip(self): return self.meta.get('trait') == 'link' # do not index links diff --git a/src/onegov/org/models/search.py b/src/onegov/org/models/search.py index 7ff35b316d..fbd25b4a2d 100644 --- a/src/onegov/org/models/search.py +++ b/src/onegov/org/models/search.py @@ -1,11 +1,17 @@ +from functools import cached_property +from operator import attrgetter + from elasticsearch_dsl.function import SF from elasticsearch_dsl.query import FunctionScore from elasticsearch_dsl.query import Match from elasticsearch_dsl.query import MatchPhrase from elasticsearch_dsl.query import MultiMatch -from functools import cached_property +from sqlalchemy import func + from onegov.core.collection import Pagination +from onegov.core.orm import Base from onegov.event.models import Event +from onegov.search.utils import searchable_sqlalchemy_models class Search(Pagination): @@ -133,3 +139,149 @@ def suggestions(self): return tuple(self.request.app.es_suggestions_by_request( self.request, self.query )) + + +def locale_mapping(locale): + mapping = {'de_CH': 'german', 'fr_CH': 'french', 'it_CH': 'italian', + 'rm_CH': 'english'} + return mapping.get(locale, 'english') + + +class SearchPostgres(Pagination): + """ + Implements searching in postgres db based on the gin index + """ + results_per_page = 10 + max_query_length = 100 + + def __init__(self, request, query, page): + self.request = request + self.query = query + self.page = page # page index + + self.nbr_of_docs = 0 + self.nbr_of_results = 0 + + @cached_property + def available_documents(self): + if not self.nbr_of_docs: + self.load_batch_results + return self.nbr_of_docs + + @cached_property + def available_results(self): + if not self.nbr_of_results: + self.load_batch_results + return self.nbr_of_results + + @property + def q(self): + return self.query + + def __eq__(self, other): + return self.page == other.page and self.query == other.query + + def subset(self): + return self.batch + + @property + def page_index(self): + return self.page + + def page_by_index(self, index): + return SearchPostgres(self.request, self.query, index) + + @cached_property + def batch(self): + if not self.query: + return None + + if self.query.startswith('#'): + results = self.hashtag_search() + else: + results = self.generic_search() + + return results[self.offset:self.offset + self.batch_size] + + @cached_property + def load_batch_results(self): + """Load search results and sort events by latest occurrence. + + This methods is a wrapper around `batch.load()`, which returns the + actual search results form the query. """ + + batch = self.batch + events = [] + non_events = [] + for search_result in batch: + if isinstance(search_result, Event): + events.append(search_result) + else: + non_events.append(search_result) + if not events: + return batch + sorted_events = sorted(events, key=lambda e: e.latest_occurrence.start) + return sorted_events + non_events + + def generic_search(self): + doc_count = 0 + results = [] + + language = locale_mapping(self.request.locale) + for model in searchable_sqlalchemy_models(Base): + if model.es_public or self.request.is_logged_in: + query = self.request.session.query(model) + doc_count += query.count() + query = query.filter( + model.fts_idx.op('@@')(func.websearch_to_tsquery( + language, self.query)) + ) + query = query.order_by(func.ts_rank_cd( + model.fts_idx, func.websearch_to_tsquery(language, + self.query))) + results.extend(query.all()) + + self.nbr_of_docs = doc_count + self.nbr_of_results = len(results) + results.sort(key=attrgetter('search_score'), reverse=False) + return results + + def hashtag_search(self): + q = self.query.lstrip('#') + results = [] + + for model in searchable_sqlalchemy_models(Base): + # skip certain tables for hashtag search for better performance + if model.__tablename__ not in ['attendees', 'files', 'people', + 'tickets', 'users']: + if model.es_public or self.request.is_logged_in: + for doc in self.request.session.query(model).all(): + if doc.es_tags and q in doc.es_tags: + results.append(doc) + + self.nbr_of_results = len(results) + results.sort(key=attrgetter('search_score'), reverse=False) + return results + + def feeling_lucky(self): + if self.batch: + first_entry = self.batch[0].load() + + # XXX the default view to the event should be doing the redirect + if first_entry.__tablename__ == 'events': + return self.request.link(first_entry, 'latest') + else: + return self.request.link(first_entry) + + @cached_property + def subset_count(self): + return self.available_results + + def suggestions(self): + suggestions = list() + + for element in self.generic_search(): + suggest = getattr(element, 'es_suggestion', []) + suggestions.append(suggest) + + return tuple(suggestions[:15]) diff --git a/src/onegov/org/path.py b/src/onegov/org/path.py index 6daf6278fd..90349572a9 100644 --- a/src/onegov/org/path.py +++ b/src/onegov/org/path.py @@ -55,6 +55,7 @@ from onegov.org.models import ResourceRecipient from onegov.org.models import ResourceRecipientCollection from onegov.org.models import Search +from onegov.org.models import SearchPostgres from onegov.org.models import SiteCollection from onegov.org.models import TicketNote from onegov.org.models import Topic @@ -506,6 +507,11 @@ def get_search(request, q='', page=0): return Search(request, q, page) +@OrgApp.path(model=SearchPostgres, path='/search_postgres') +def get_postgres_search(request, q='', page=0): + return SearchPostgres(request, q, page) + + @OrgApp.path(model=AtoZPages, path='/a-z') def get_a_to_z(request): return AtoZPages(request) diff --git a/src/onegov/org/templates/search_postgres.pt b/src/onegov/org/templates/search_postgres.pt new file mode 100644 index 0000000000..fd6a15c450 --- /dev/null +++ b/src/onegov/org/templates/search_postgres.pt @@ -0,0 +1,69 @@ +
+ + ${title} + + +
+
+ Postgres Searching is currently unavailable due to technical + difficulties. + Please excuse the inconvenience and try again later. +
+
+ +
+
+
+ +
+ +
+ +
+
+
+
+
+
+
+
+ + +

${resultslabel}

+
+
+

Your postgres search returned no results.

+ +
    + +
  • + + + +
      +
    • Score: ${result.explanation['score']}
      • + +
    • + ${title}: ${result.explanation[key]['value']} +
      • +       +
    +     +
    • +     +
+ + +
+ +
+
+ + +
diff --git a/src/onegov/org/views/search.py b/src/onegov/org/views/search.py index c172c66201..35e985fdb1 100644 --- a/src/onegov/org/views/search.py +++ b/src/onegov/org/views/search.py @@ -4,14 +4,13 @@ from onegov.org import _, OrgApp from onegov.org.elements import Link from onegov.org.layout import DefaultLayout -from onegov.org.models import Search +from onegov.org.models import Search, SearchPostgres from onegov.search import SearchOfflineError from webob import exc @OrgApp.html(model=Search, template='search.pt', permission=Public) def search(self, request, layout=None): - layout = layout or DefaultLayout(self, request) layout.breadcrumbs.append(Link(_("Search"), '#')) @@ -46,9 +45,56 @@ def search(self, request, layout=None): } +@OrgApp.html(model=SearchPostgres, template='search_postgres.pt', + permission=Public) +def search_postgres(self, request, layout=None): + layout = layout or DefaultLayout(self, request) + layout.breadcrumbs.append(Link(_("Search"), '#')) + + try: + searchlabel = _("Search through ${count} indexed documents", mapping={ + 'count': self.available_documents + }) + resultslabel = _("${count} Results", mapping={ + 'count': self.available_results + }) + except SearchOfflineError: + return { + 'title': _("Search Unavailable"), + 'layout': layout, + 'connection': False + } + + if 'lucky' in request.GET: + url = self.feeling_lucky() + + if url: + return morepath.redirect(url) + + return { + # TODO switch back to 'Search' once es is gone + # 'title': _("Search"), + 'title': _("Org Search Postgres"), + 'model': self, + 'layout': layout, + 'hide_search_header': True, + 'searchlabel': searchlabel, + 'resultslabel': resultslabel, + 'connection': True + } + + @OrgApp.json(model=Search, name='suggest', permission=Public) def suggestions(self, request): try: return tuple(self.suggestions()) except SearchOfflineError as exception: raise exc.HTTPNotFound() from exception + + +@OrgApp.json(model=SearchPostgres, name='suggest', permission=Public) +def suggestions_postgres(self, request): + try: + return tuple(self.suggestions()) + except SearchOfflineError as exception: + raise exc.HTTPNotFound() from exception diff --git a/src/onegov/page/model.py b/src/onegov/page/model.py index 1a2fd7bec6..0d88ceb4f0 100644 --- a/src/onegov/page/model.py +++ b/src/onegov/page/model.py @@ -25,6 +25,10 @@ class Page(AdjacencyList, ContentMixin, TimestampMixin, UTCPublicationMixin): __tablename__ = 'pages' + @property + def search_score(self) -> int: + return 2 + if TYPE_CHECKING: # we override these relationships to be more specific parent: relationship['Page'] diff --git a/src/onegov/people/models/membership.py b/src/onegov/people/models/membership.py index b1e3d41de7..46efeebe03 100644 --- a/src/onegov/people/models/membership.py +++ b/src/onegov/people/models/membership.py @@ -88,6 +88,10 @@ class AgencyMembership(Base, ContentMixin, TimestampMixin, ORMSearchable, #: when the membership started since = Column(Text, nullable=True) + @property + def search_score(self): + return 3 + @property def siblings_by_agency(self): """ Returns a query that includes all siblings by agency, including diff --git a/src/onegov/people/models/person.py b/src/onegov/people/models/person.py index 0da149828e..6acc883628 100644 --- a/src/onegov/people/models/person.py +++ b/src/onegov/people/models/person.py @@ -1,11 +1,12 @@ +from sqlalchemy.ext.hybrid import hybrid_property + from onegov.core.orm import Base from onegov.core.orm.mixins import ContentMixin from onegov.core.orm.mixins import TimestampMixin from onegov.core.orm.mixins import UTCPublicationMixin from onegov.core.orm.types import UUID from onegov.search import ORMSearchable -from sqlalchemy import Column -from sqlalchemy import Text +from sqlalchemy import Column, Text from uuid import uuid4 from vobject import vCard from vobject.vcard import Address @@ -38,9 +39,9 @@ class Person(Base, ContentMixin, TimestampMixin, ORMSearchable, @property def es_suggestion(self): - return (self.title, f"{self.first_name} {self.last_name}") + return self.title - @property + @hybrid_property def title(self): """ Returns the Estern-ordered name. """ @@ -129,6 +130,10 @@ def spoken_title(self): #: some remarks about the person notes = Column(Text, nullable=True) + @property + def search_score(self): + return 3 + def vcard_object(self, exclude=None, include_memberships=True): """ Returns the person as vCard (3.0) object. diff --git a/src/onegov/search/__init__.py b/src/onegov/search/__init__.py index 24e6219b8d..f71bf05d93 100644 --- a/src/onegov/search/__init__.py +++ b/src/onegov/search/__init__.py @@ -4,11 +4,11 @@ from onegov.search.mixins import Searchable, ORMSearchable, SearchableContent from onegov.search.dsl import Search -from onegov.search.integration import ElasticsearchApp +from onegov.search.integration import SearchApp from onegov.search.errors import SearchOfflineError __all__ = [ - 'ElasticsearchApp', + 'SearchApp', 'ORMSearchable', 'Search', 'Searchable', diff --git a/src/onegov/search/cli.py b/src/onegov/search/cli.py index c87cc86147..9b459f2620 100644 --- a/src/onegov/search/cli.py +++ b/src/onegov/search/cli.py @@ -5,7 +5,6 @@ from onegov.core.cli import command_group, pass_group_context from sedate import utcnow - cli = command_group() @@ -13,18 +12,26 @@ @click.option('--fail', is_flag=True, default=False, help='Fail on errors') @pass_group_context def reindex(group_context, fail): - """ Reindexes all objects in the elasticsearch database. """ + """ Reindexes all objects in the postgresql database. """ def run_reindex(request, app): - if not hasattr(request.app, 'es_client'): - return + """ + Looping over all models in project deleting all full text search ( + fts) indexes in postgresql and re-creating them + :param request: request + :param app: application context + """ title = f"Reindexing {request.app.application_id}" print(click.style(title, underline=True)) start = utcnow() - request.app.es_perform_reindex(fail) + app.psql_perform_reindex(request) + print(f"- psql indexing took {utcnow() - start}") - print(f"took {utcnow() - start}") + # TODO: remove es indexing once es is gone + start = utcnow() + request.app.es_perform_reindex(fail) + print(f"- es indexing took {utcnow() - start}") return run_reindex diff --git a/src/onegov/search/integration.py b/src/onegov/search/integration.py index 3880302fa2..adc973b133 100644 --- a/src/onegov/search/integration.py +++ b/src/onegov/search/integration.py @@ -10,6 +10,8 @@ from elasticsearch import TransportError from elasticsearch.connection import create_ssl_context from more.transaction.main import transaction_tween_factory + +from onegov.core.orm import Base from onegov.search import Search, log from onegov.search.errors import SearchOfflineError from onegov.search.indexer import Indexer @@ -90,8 +92,9 @@ def is_5xx_error(error): return error.status_code and str(error.status_code).startswith('5') -class ElasticsearchApp(morepath.App): - """ Provides elasticsearch integration for +# TODO: remove all es specific things ones es is gone +class SearchApp(morepath.App): + """ Provides elasticsearch and postgres integration for :class:`onegov.core.framework.Framework` based applications. The application must be connected to a database. @@ -326,6 +329,7 @@ def es_suggestions_by_request(self, request, query, types='*', else: languages = '*' + print(f'es_suggestion_by_request language: {languages}') return self.es_suggestions( query, languages=languages, @@ -394,8 +398,20 @@ def reindex_model(model): self.es_indexer.bulk_process() + def psql_perform_reindex(self, request): + """ Re-indexes all `searchable' models in postgresql db ensuring + each table will be indexed only once. + + """ + done = [] + + for model in searchable_sqlalchemy_models(Base): + if model.__tablename__ not in done: + model.reindex(request, model) + done.append(model.__tablename__) + -@ElasticsearchApp.tween_factory(over=transaction_tween_factory) +@SearchApp.tween_factory(over=transaction_tween_factory) def process_indexer_tween_factory(app, handler): def process_indexer_tween(request): diff --git a/src/onegov/search/mixins.py b/src/onegov/search/mixins.py index e6cfb01149..b1f2b644fc 100644 --- a/src/onegov/search/mixins.py +++ b/src/onegov/search/mixins.py @@ -1,6 +1,15 @@ -from onegov.search.utils import classproperty +from sqlalchemy import Column, func, Computed # type:ignore[attr-defined] +from sqlalchemy.dialects.postgresql import TSVECTOR +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.orm import deferred + +from onegov.core.upgrade import UpgradeContext +from onegov.search.utils import classproperty, \ + get_fts_index_localized_languages, get_fts_index_basic_languages from onegov.search.utils import extract_hashtags +from typing import Any, TYPE_CHECKING + class Searchable: """ Defines the interface required for an object to be searchable. @@ -38,6 +47,18 @@ def es_type_name(self): identity is a completely different model. """ + TEXT_SEARCH_COLUMN_NAME = 'fts_idx' + + if TYPE_CHECKING: + fts_idx: 'Column[dict[str, Any]]' + + # column for full text search index + @declared_attr # type:ignore[no-redef] + def fts_idx(cls) -> 'Column[dict[str, Any]]': + col_name = Searchable.TEXT_SEARCH_COLUMN_NAME + if hasattr(cls, '__table__') and hasattr(cls.__table__.c, col_name): + return deferred(cls.__table__.c.fts_idx) + return deferred(Column(col_name, TSVECTOR)) @classproperty def es_properties(self): @@ -134,7 +155,106 @@ def es_tags(self): """ Returns a list of tags associated with this content. """ return None + @property + def search_score(self): + """ + the lower the score the higher the class type will be shown in search + results. Default is 10 (lowest) + """ + return 10 + + @staticmethod + def psql_tsvector_expression(model): + """ + Provides the tsvector expression for postgres for the defined + model. Depending on the model columns and properties are used for full + text search index. + + :return: tsvector expression + """ + objects = [getattr(model, p) for p in model.es_properties if + not p.startswith('es_')] + return Searchable.create_tsvector_expression(*objects) + + @staticmethod + def reindex(request, model): + """ + Re-indexes the table by dropping and adding the full text search + column. + """ + Searchable.drop_fts_column(request, model) + Searchable.add_fts_column(request, model) + + @staticmethod + def drop_fts_column(request, model): + """ + Drops the full text search column + + :param request: request object + :param model: model to drop the index from + :return: None + """ + + col_name = Searchable.TEXT_SEARCH_COLUMN_NAME + context = UpgradeContext(request) + + if context.has_column(model.__tablename__, col_name): + context.operations.drop_column(model.__tablename__, col_name) + + @staticmethod + def add_fts_column(request, model): + + """ + This function is used for re-indexing and as migration step moving to + postgresql full text search (fts), OGC-508. + + It adds a separate column for the tsvector to `schema`.`table` + creating a multilingual gin index on the columns/data defined per + model. + + :param request: request object + :param model: model to add the index + :return: None + """ + col_name = Searchable.TEXT_SEARCH_COLUMN_NAME + context = UpgradeContext(request) + if not context.has_column(model.__tablename__, col_name): + tsvector_expression = None + for prop_name, type_info in model.es_properties.items(): + if not prop_name.startswith('es_'): + prop_type = type_info.get('type', None) + prop = getattr(model, prop_name) + languages = get_fts_index_basic_languages() + + if prop_type in ['localized', 'localized_html']: + # only for 'localized' properties we create the + # index localized + languages.extend(get_fts_index_localized_languages()) + + for language in languages: + expr = func.to_tsvector(language, + func.coalesce(prop, '')) + + if tsvector_expression is None: + tsvector_expression = expr + else: + tsvector_expression = tsvector_expression.concat( + expr) + + context.operations.add_column( + model.__tablename__, + Column(col_name, + TSVECTOR, + Computed( + tsvector_expression, + persisted=True), + ) + ) + context.operations.execute("COMMIT") + + +# TODO: rename prefix 'es' to 'ts' for text search class ORMSearchable(Searchable): """ Extends the default :class:`Searchable` class with sensible defaults for SQLAlchemy orm models. @@ -154,6 +274,7 @@ def es_last_change(self): return getattr(self, 'last_change', None) +# TODO: rename prefix 'es' to 'ts' for text search class SearchableContent(ORMSearchable): """ Adds search to all classes using the core's content mixin: :class:`onegov.core.orm.mixins.content.ContentMixin` diff --git a/src/onegov/search/utils.py b/src/onegov/search/utils.py index 8d02ebe00e..1457251f58 100644 --- a/src/onegov/search/utils.py +++ b/src/onegov/search/utils.py @@ -8,7 +8,6 @@ from langdetect.utils.lang_profile import LangProfile from onegov.core.orm import find_models - # XXX this is doubly defined in onegov.org.utils, maybe move to a common # regex module in in onegov.core HASHTAG = re.compile(r'#\w{3,}') @@ -30,6 +29,16 @@ def searchable_sqlalchemy_models(base): _invalid_index_characters = re.compile(r'[\\/?"<>|\s,A-Z:]+') +def get_fts_index_languages(): + """ Define index creation languages for full text search as we have a + limited set of used languages. + + NOTE: 'simple' is used for tag, label or phrase searches + + """ + return ['simple', 'german', 'french', 'italian', 'english'] + + def is_valid_index_name(name): """ Checks if the given name is a valid elasticsearch index name. Elasticsearch does it's own checks, but we can do it earlier and we are diff --git a/src/onegov/ticket/model.py b/src/onegov/ticket/model.py index 538f3e9b79..885045130e 100644 --- a/src/onegov/ticket/model.py +++ b/src/onegov/ticket/model.py @@ -1,3 +1,5 @@ +from sqlalchemy.ext.hybrid import hybrid_property + from onegov.core.orm import Base from onegov.core.orm.mixins import TimestampMixin from onegov.core.orm.types import JSON, UUID @@ -105,6 +107,10 @@ class Ticket(Base, TimestampMixin, ORMSearchable): #: true if the notifications for this ticket should be muted muted: 'Column[bool]' = Column(Boolean, nullable=False, default=False) + @property + def search_score(self) -> int: + return 6 + if TYPE_CHECKING: created: Column[datetime] else: @@ -142,6 +148,7 @@ def created(cls) -> 'Column[datetime]': # limit the search to the ticket number -> the rest can be found es_public = False + es_properties = { 'number': {'type': 'text'}, 'title': {'type': 'text'}, @@ -152,7 +159,7 @@ def created(cls) -> 'Column[datetime]': 'extra_localized_text': {'type': 'localized'} } - @property + @hybrid_property def extra_localized_text(self) -> str | None: """ Maybe used by child-classes to return localized extra data that should be indexed as well. @@ -167,14 +174,14 @@ def es_suggestion(self) -> list[str]: self.number.replace('-', '') ] - @property + @hybrid_property def ticket_email(self) -> str | None: if self.handler.deleted: return self.snapshot.get('email') else: return self.handler.email - @property + @hybrid_property def ticket_data(self) -> 'Sequence[str] | None': if self.handler.deleted: return self.snapshot.get('summary') diff --git a/src/onegov/town6/templates/search_postgres.pt b/src/onegov/town6/templates/search_postgres.pt new file mode 100644 index 0000000000..055758f94a --- /dev/null +++ b/src/onegov/town6/templates/search_postgres.pt @@ -0,0 +1,70 @@ +
+ + ${title} + + +
+
+ Postgres Searching is currently unavailable due to technical + difficulties. + Please excuse the inconvenience and try again later. +
+
+ +
+
+
+ +
+ +
+ +
+
+
+
+
+
+
+
+ + +

${resultslabel}

+
+
+ +

Your postgres search returned no results.

+ +
    + +
  • + + + +
      +
    • Score: ${result.explanation['score']}
      • + +
    • + ${title}: ${result.explanation[key]['value']} +
      • +       +
    +     +
    • +     +
+ + +
+ +
+
+ + +
diff --git a/src/onegov/town6/views/search.py b/src/onegov/town6/views/search.py index 6e7b1d71d4..14d5bdc62d 100644 --- a/src/onegov/town6/views/search.py +++ b/src/onegov/town6/views/search.py @@ -1,10 +1,16 @@ from onegov.core.security import Public -from onegov.org.views.search import search +from onegov.org.views.search import search, search_postgres from onegov.town6 import TownApp -from onegov.org.models import Search +from onegov.org.models import Search, SearchPostgres from onegov.town6.layout import DefaultLayout @TownApp.html(model=Search, template='search.pt', permission=Public) def town_search(self, request): return search(self, request, DefaultLayout(self, request)) + + +@TownApp.html(model=SearchPostgres, template='search_postgres.pt', + permission=Public) +def town_search_postgres(self, request): + return search_postgres(self, request, DefaultLayout(self, request)) diff --git a/src/onegov/translator_directory/models/translator.py b/src/onegov/translator_directory/models/translator.py index 82b5e60a14..53f4dcfd60 100644 --- a/src/onegov/translator_directory/models/translator.py +++ b/src/onegov/translator_directory/models/translator.py @@ -28,6 +28,7 @@ from .language import Language +# TODO rename to ts (text search) class ESMixin(ORMSearchable): es_properties = { @@ -182,6 +183,10 @@ class Translator(Base, TimestampMixin, AssociatedFiles, ContentMixin, expertise_professional_guilds_other: 'dict_property[Sequence[str]]' expertise_professional_guilds_other = meta_property(default=tuple) + @property + def search_score(self): + return 4 + @property def expertise_professional_guilds_all(self): return ( diff --git a/src/onegov/user/models/user.py b/src/onegov/user/models/user.py index c1eee656c1..70eb73f36d 100644 --- a/src/onegov/user/models/user.py +++ b/src/onegov/user/models/user.py @@ -1,4 +1,5 @@ from datetime import datetime + from onegov.core.crypto import hash_password, verify_password from onegov.core.orm import Base from onegov.core.orm.mixins import data_property, dict_property, TimestampMixin @@ -9,8 +10,7 @@ from onegov.core.utils import yubikey_otp_to_serial from onegov.search import ORMSearchable from onegov.user.models.group import UserGroup -from sqlalchemy import Boolean, Column, Index, Text, func, ForeignKey -from sqlalchemy import UniqueConstraint +from sqlalchemy import Boolean, Column, Text, func, ForeignKey from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import backref, deferred, relationship from uuid import uuid4, UUID as UUIDType @@ -57,7 +57,7 @@ class User(Base, TimestampMixin, ORMSearchable): def es_suggestion(self) -> tuple[str, str]: return (self.realname or self.username, self.username) - @property + @hybrid_property def userprofile(self) -> list[str]: if not self.data: return [] @@ -150,10 +150,9 @@ def userprofile(self) -> list[str]: signup_token: 'Column[str | None]' = Column( Text, nullable=True, default=None) - __table_args__ = ( - Index('lowercase_username', func.lower(username), unique=True), - UniqueConstraint('source', 'source_id', name='unique_source_id'), - ) + @property + def search_score(self) -> int: + return 5 if TYPE_CHECKING: # HACK: This probably won't be necessary in SQLAlchemy 2.0, but diff --git a/tests/onegov/search/test_indexer.py b/tests/onegov/search/test_indexer.py index 8a1fe21104..3ad31b52bb 100644 --- a/tests/onegov/search/test_indexer.py +++ b/tests/onegov/search/test_indexer.py @@ -2,6 +2,11 @@ import pytest from datetime import datetime + +from onegov.core.orm import Base +from onegov.directory import DirectoryEntry +from onegov.org.models import Topic +from onegov.people import Agency from onegov.search import Searchable, SearchOfflineError, utils from onegov.search.indexer import parse_index_name from onegov.search.indexer import ( @@ -14,6 +19,10 @@ from queue import Queue from unittest.mock import Mock +from onegov.search.utils import searchable_sqlalchemy_models +from onegov.ticket import Ticket +from onegov.user import User + def test_index_manager_assertions(es_client): @@ -717,3 +726,88 @@ def test_elasticsearch_outage(es_client, es_url): indexer.es_client.indices.refresh(index='_all') assert indexer.es_client\ .search(index='_all')['hits']['total']['value'] == 2 + + +def test_psql_tsvector_string(): + assert Searchable.create_tsvector_string(('col_lower')) == \ + "'func.coalesce(col_lower, '')'" + + # FIXME: implement lower + # assert Searchable.create_tsvector_string(['Col_Higher']) == \ + # 'coalesce("\'col_higher\'", \'\')' + + assert Searchable.create_tsvector_string('col_a', 'col_b') == \ + "'func.coalesce(col_a, '')' || ' ' || 'func.coalesce(col_b, '')'" + + assert Searchable.create_tsvector_string('a', 'b', 'c') == \ + "'func.coalesce(a, '')' || ' ' || 'func.coalesce(b, '')' || ' ' || " \ + "'func.coalesce(c, '')'" + + cols = ['col_a', 'col_b'] + assert Searchable.create_tsvector_string(*cols) == \ + "'func.coalesce(col_a, '')' || ' ' || 'func.coalesce(col_b, '')'" + + +def test_multi_language_tsvector_expression(monkeypatch): + tsvector_string = "'func.coalesce(my_col, '')'" + x = Searchable.multi_language_tsvector_expression(tsvector_string) + assert x == "to_tsvector('simple', 'func.coalesce(my_col, '')') || " \ + "to_tsvector('german', 'func.coalesce(my_col, '')') || " \ + "to_tsvector('french', 'func.coalesce(my_col, '')') || " \ + "to_tsvector('italian', 'func.coalesce(my_col, '')') || " \ + "to_tsvector('english', 'func.coalesce(my_col, '')')" + + def fake(): + return ['simple'] + tsvector_string = "'func.coalesce(group, '')'" + monkeypatch.setattr(utils, 'get_fts_index_languages', fake) + assert Searchable.multi_language_tsvector_expression( + tsvector_string) == "to_tsvector('simple', 'func.coalesce(group, '')')" + + +def test_psql_tsvector_string_generation_models(): + count = 0 + + for model in searchable_sqlalchemy_models(Base): + print(f'model {model}..') + tsvector = model.psql_tsvector_string(model) + for p in getattr(model, 'es_properties', []): + if p in model.__dict__ and not p.startswith('_es'): + # verify all properties are reflected in the tsvector + assert p in tsvector + + # random sample + if model == Agency: + count += 1 + assert tsvector == "'func.coalesce(title, '')' || ' ' || " \ + "'func.coalesce(description, '')' || ' ' || " \ + "'func.coalesce(portrait, '')'" + elif model == User: + count += 1 + assert tsvector == "'func.coalesce(username, '')' || ' ' || " \ + "'func.coalesce(realname, '')' || ' ' || " \ + "'func.coalesce(userprofile, '')'" + elif model == DirectoryEntry: + count += 1 + assert tsvector == "'func.coalesce(keywords, '')' || ' ' || " \ + "'func.coalesce(title, '')' || ' ' || " \ + "'func.coalesce(lead, '')' || ' ' || " \ + "'func.coalesce(directory_id, '')' || ' ' || " \ + "'func.coalesce(text, '')'" + elif model == Ticket: + count += 1 + assert tsvector == "'func.coalesce(number, '')' || ' ' || " \ + "'func.coalesce(title, '')' || ' ' || " \ + "'func.coalesce(subtitle, '')' || ' ' || " \ + "'func.coalesce(group, '')' || ' ' || " \ + "'func.coalesce(ticket_email, '')' || ' ' || " \ + "'func.coalesce(ticket_data, '')' || ' ' || " \ + "'func.coalesce(extra_localized_text, '')'" + elif model == Topic: + count += 1 + assert tsvector == "'func.coalesce(title, '')' || ' ' || " \ + "'func.coalesce(lead, '')' || ' ' || " \ + "'func.coalesce(text, '')'" + + # verify if statements reached and tested + assert count == 5 diff --git a/tests/onegov/search/test_integration.py b/tests/onegov/search/test_integration.py index 3280604079..a018434c6c 100644 --- a/tests/onegov/search/test_integration.py +++ b/tests/onegov/search/test_integration.py @@ -5,10 +5,11 @@ from datetime import timedelta from elasticsearch_dsl.function import SF from elasticsearch_dsl.query import MatchPhrase, FunctionScore + from onegov.core import Framework from onegov.core.orm.mixins import TimestampMixin from onegov.core.utils import scan_morepath_modules -from onegov.search import ElasticsearchApp, ORMSearchable +from onegov.search import SearchApp, ORMSearchable from sqlalchemy import Boolean, Column, Integer, Text from sqlalchemy.ext.declarative import declarative_base from webtest import TestApp as Client @@ -17,7 +18,7 @@ def test_app_integration(es_url): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass app = App() @@ -33,7 +34,7 @@ class App(Framework, ElasticsearchApp): def test_search_query(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -46,12 +47,20 @@ class Document(Base, ORMSearchable): body = Column(Text, nullable=True) public = Column(Boolean, nullable=False) language = Column(Text, nullable=False) + # fts_idx = Column(TSVECTOR, Computed('', persisted=True)) + # __table_args__ = ( + # Index('fts_idx', fts_idx, postgresql_using='gin'), + # ) es_properties = { 'title': {'type': 'localized'}, 'body': {'type': 'localized'} } + # @staticmethod + # def psql_tsvector_string(): + # return Searchable.create_tsvector_string('title', 'body') + @property def es_suggestion(self): return self.title @@ -148,10 +157,17 @@ def es_language(self): assert document.title == "Öffentlich" assert document.public + ################## + # postgresql tests + # app.psql_perform_reindex() + + # results = app.psql_search('') + # assert results + def test_orm_integration(es_url, postgres_dsn, redis_url): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -294,7 +310,7 @@ def view_delete_document(self, request): def test_alternate_id_property(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -356,7 +372,7 @@ def es_suggestion(self): def test_orm_polymorphic(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -442,7 +458,7 @@ def update(): def test_orm_polymorphic_sublcass_only(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -498,7 +514,7 @@ def es_suggestion(self): def test_suggestions(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -617,7 +633,7 @@ def es_suggestion(self): def test_language_detection(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -671,7 +687,7 @@ class Document(Base, ORMSearchable): def test_language_update(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() @@ -725,7 +741,7 @@ class Document(Base, ORMSearchable): def test_date_decay(es_url, postgres_dsn): - class App(Framework, ElasticsearchApp): + class App(Framework, SearchApp): pass Base = declarative_base() diff --git a/tests/onegov/search/test_utils.py b/tests/onegov/search/test_utils.py index f5c29c3a97..fb88ea10e4 100644 --- a/tests/onegov/search/test_utils.py +++ b/tests/onegov/search/test_utils.py @@ -1,5 +1,6 @@ -from onegov.search import ORMSearchable, Searchable +from onegov.search import ORMSearchable from onegov.search import utils +from onegov.search.mixins import Searchable from sqlalchemy import Column, Integer, Text from sqlalchemy.ext.declarative import declarative_base @@ -101,3 +102,13 @@ class News(Page): es_type_name = 'news' assert utils.related_types(Page) == {'news', 'topic'} + + +def test_create_tsvector_string(): + assert Searchable.create_tsvector_string('username') == \ + "coalesce(username, '')" + assert Searchable.create_tsvector_string('title', 'body') == \ + "coalesce(title, '') || ' ' || coalesce(body, '')" + assert Searchable.create_tsvector_string('alpha', 'beta', 'gamma') == \ + "coalesce(alpha, '') || ' ' || coalesce(beta, '') || ' ' || " \ + "coalesce(gamma, '')" diff --git a/tests/shared/fixtures.py b/tests/shared/fixtures.py index 5d911910e5..5ecf246805 100644 --- a/tests/shared/fixtures.py +++ b/tests/shared/fixtures.py @@ -36,7 +36,7 @@ from threading import Thread from uuid import uuid4 from webdriver_manager.chrome import ChromeDriverManager -from webdriver_manager.core.os_manager import ChromeType +# from webdriver_manager.core.os_manager import ChromeType redis_path = which('redis-server')