Feature/ogc 508 replace elastic search by postgres text search

src/onegov/activity/models/attendee.py

@@ -1,4 +1,5 @@
 from datetime import date
+
 from onegov.activity.models.booking import Booking
 from onegov.core.orm import Base
 from onegov.core.orm.mixins import TimestampMixin
@@ -11,7 +12,6 @@
 from sqlalchemy import Date
 from sqlalchemy import Float
 from sqlalchemy import ForeignKey
-from sqlalchemy import Index
 from sqlalchemy import Integer
 from sqlalchemy import Numeric
 from sqlalchemy import Text
@@ -43,6 +43,10 @@ class Attendee(Base, TimestampMixin, ORMSearchable):
     }
     es_public = False
 
+    @property
+    def search_score(self):
+        return 3
+
     @property
     def es_suggestion(self):
         return self.name
@@ -179,7 +183,3 @@ def happiness(cls, period_id):
         order_by='Booking.created',
         backref='attendee'
     )
-
-    __table_args__ = (
-        Index('unique_child_name', 'username', 'name', unique=True),
-    )

src/onegov/agency/models/person.py

@@ -21,8 +21,6 @@ def es_public(self):
         'title': {'type': 'text'},
         'function': {'type': 'localized'},
         'email': {'type': 'text'},
-        'phone_internal': {'type': 'text'},
-        'phone_es': {'type': 'text'}
     }
 
     @property

src/onegov/agency/views/search.py

@@ -1,13 +1,23 @@
 from onegov.agency import AgencyApp
 from onegov.agency.layout import AgencySearchLayout
 from onegov.core.security import Public
-from onegov.org.models import Search
+from onegov.org.models import Search, SearchPostgres
 from onegov.org.views.search import search as search_view
+from onegov.org.views.search import search_postgres as search_postgres_view
 
 
 @AgencyApp.html(model=Search, template='search.pt', permission=Public)
-def search(self, request):
+def agency_search(self, request):
     data = search_view(self, request)
     if isinstance(data, dict):
         data['layout'] = AgencySearchLayout(self, request)
     return data
+
+
+@AgencyApp.html(model=SearchPostgres, template='search_postgres.pt',
+                permission=Public)
+def agency_search_postgres(self, request):
+    data = search_postgres_view(self, request)
+    if isinstance(data, dict):
+        data['layout'] = AgencySearchLayout(self, request)
+    return data

src/onegov/core/upgrade.py

@@ -480,6 +480,12 @@ def has_column(self, table: str, column: str) -> bool:
             table, schema=self.schema
         )}
 
+    def has_index(self, table: str, index: str) -> bool:
+        inspector = Inspector(self.operations_connection)
+        return index in {i['name'] for i in inspector.get_indexes(
+            table, schema=self.schema
+        )}
+
     def has_enum(self, enum: str) -> bool:
         return self.session.execute(f"""
             SELECT EXISTS (

src/onegov/directory/models/directory.py

@@ -17,10 +17,8 @@
 from onegov.file.utils import as_fileintent
 from onegov.form import flatten_fieldsets, parse_formcode, parse_form
 from onegov.search import SearchableContent
-from sqlalchemy import Column
+from sqlalchemy import Column, Text, Integer
 from sqlalchemy import func, exists, and_
-from sqlalchemy import Integer
-from sqlalchemy import Text
 from sqlalchemy.orm import object_session
 from sqlalchemy.orm import relationship
 from sqlalchemy.orm.attributes import InstrumentedAttribute
@@ -103,6 +101,10 @@ def count(self):
         backref='directory'
     )
 
+    @property
+    def search_score(self):
+        return 7
+
     @property
     def entry_cls_name(self):
         return 'DirectoryEntry'

src/onegov/directory/models/directory_entry.py

@@ -6,12 +6,13 @@
 from onegov.file import AssociatedFiles
 from onegov.gis import CoordinatesMixin
 from onegov.search import SearchableContent
-from sqlalchemy import Column
+from sqlalchemy import Column, cast
 from sqlalchemy import ForeignKey
 from sqlalchemy import Index
 from sqlalchemy import Text
 from sqlalchemy.dialects.postgresql import HSTORE
 from sqlalchemy.ext.mutable import MutableDict
+from sqlalchemy.ext.hybrid import hybrid_property
 from uuid import uuid4
 
 
@@ -25,7 +26,7 @@ class DirectoryEntry(Base, ContentMixin, CoordinatesMixin, TimestampMixin,
         'keywords': {'type': 'keyword'},
         'title': {'type': 'localized'},
         'lead': {'type': 'localized'},
-        'directory_id': {'type': 'keyword'},
+        '_directory_id': {'type': 'keyword'},
 
         # since the searchable text might include html, we remove it
         # even if there's no html -> possibly decreasing the search
@@ -86,6 +87,10 @@ def external_link_title(self):
     def external_link_visible(self):
         return self.directory.configuration.link_visible
 
+    @hybrid_property
+    def _directory_id(self):
+        return cast(self.directory_id, Text)
+
     @property
     def directory_name(self):
         return self.directory.name
@@ -98,7 +103,7 @@ def keywords(self):
     def keywords(self, value):
         self._keywords = {k: '' for k in value} if value else None
 
-    @property
+    @hybrid_property
     def text(self):
         return self.directory.configuration.extract_searchable(self.values)
 

src/onegov/event/models/event.py

@@ -6,6 +6,7 @@
 from icalendar import Calendar as vCalendar
 from icalendar import Event as vEvent
 from icalendar import vRecur
+from sqlalchemy.ext.hybrid import hybrid_property
 
 from onegov.core.orm import Base
 from onegov.core.orm.abstract import associated
@@ -109,6 +110,10 @@ class Event(Base, OccurrenceMixin, TimestampMixin, SearchableContent,
         EventFile, 'pdf', 'one-to-one', uselist=False, backref_suffix='pdf'
     )
 
+    @property
+    def search_score(self):
+        return 1
+
     def set_image(self, content, filename=None):
         self.set_blob('image', content, filename)
 
@@ -148,9 +153,16 @@ def set_blob(self, blob, content, filename=None):
         'description': {'type': 'localized'},
         'location': {'type': 'localized'},
         'organizer': {'type': 'localized'},
-        'filter_keywords': {'type': 'keyword'}
     }
 
+    @hybrid_property
+    def description(self):  # noqa: F811
+        return self.content['description'].astext
+
+    @hybrid_property
+    def organizer(self):  # noqa: F811
+        return self.content['organizer'].astext
+
     @property
     def es_public(self):
         return self.state == 'published'

src/onegov/feriennet/models/activity.py

@@ -1,4 +1,7 @@
 from functools import cached_property
+
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.activity import Activity, ActivityCollection, Occasion
 from onegov.activity import PublicationRequestCollection
 from onegov.activity.models import DAYS
@@ -24,6 +27,10 @@ class VacationActivity(Activity, CoordinatesExtension, SearchableContent):
         'organiser': {'type': 'text'}
     }
 
+    @property
+    def search_score(self):
+        return 1
+
     @property
     def es_public(self):
         return self.state == 'accepted'
@@ -32,7 +39,7 @@ def es_public(self):
     def es_skip(self):
         return self.state == 'preview'
 
-    @property
+    @hybrid_property
     def organiser(self):
         organiser = [
             self.user.username,

src/onegov/feriennet/templates/mail_booking_accepted.pt

@@ -16,7 +16,7 @@
             <tal:b tal:content="structure bookings_link"></tal:b>
         </p>
         <br>
-        <p i18n:translate>Best regards</p>                    
+        <p i18n:translate>Best regards</p>
         <metal:b use-macro="layout.macros['sender']|nothing" />
     </tal:b>
 </div>

src/onegov/feriennet/templates/mail_booking_canceled.pt

@@ -15,7 +15,7 @@
             <tal:b tal:content="structure bookings_link"></tal:b>
         </p>
         <br>
-        <p i18n:translate>Best regards</p>                    
+        <p i18n:translate>Best regards</p>
         <metal:b use-macro="layout.macros['sender']|nothing" />
     </tal:b>
 </div>

src/onegov/file/models/file.py

@@ -6,6 +6,7 @@
 from contextlib import contextmanager
 from collections import defaultdict
 from depot.fields.sqlalchemy import UploadedFileField as UploadedFileFieldBase
+
 from onegov.core.crypto import random_token
 from onegov.core.orm import Base
 from onegov.core.orm.abstract import Associable
@@ -262,6 +263,10 @@ class File(Base, Associable, TimestampMixin):
         Index('files_by_type_and_order', 'type', 'order'),
     )
 
+    @property
+    def search_score(self) -> int:
+        return 10
+
     @hybrid_property
     def signature_timestamp(self) -> 'datetime | None':
         if self.signed:

src/onegov/file/models/fileset.py

@@ -6,6 +6,7 @@
 
 
 from typing import TYPE_CHECKING
+
 if TYPE_CHECKING:
     from .file import File
 

src/onegov/form/models/definition.py

@@ -140,6 +140,9 @@ class FormDefinition(Base, ContentMixin, TimestampMixin, Extendable):
     }
 
     @property
+    def search_score(self) -> int:
+        return 7
+
     def form_class(self) -> Type['Form']:
         """ Parses the form definition and returns a form class. """
 

src/onegov/fsi/models/course.py

@@ -1,4 +1,5 @@
 from arrow import utcnow
+
 from onegov.core.html import html_to_text
 from onegov.core.orm import Base
 from onegov.core.orm.types import UUID
@@ -32,6 +33,10 @@ class Course(Base, ORMSearchable):
     # hides the course in the collection for non-admins
     hidden_from_public = Column(Boolean, nullable=False, default=False)
 
+    @property
+    def search_score(self):
+        return 2
+
     @property
     def title(self):
         return self.name

src/onegov/fsi/models/course_attendee.py

@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.core.orm import Base
 from onegov.core.orm.types import UUID, JSON
 from sqlalchemy import Boolean
@@ -107,7 +109,7 @@ def __str__(self):
         cascade='all, delete-orphan'
     )
 
-    @property
+    @hybrid_property
     def title(self):
         return ' '.join((
             p for p in (
@@ -131,7 +133,7 @@ def role(self):
             return 'member'
         return self.user.role
 
-    @property
+    @hybrid_property
     def email(self):
         """Needs a switch for external users"""
         if not self.user_id:

src/onegov/fsi/models/course_event.py

@@ -22,13 +22,15 @@
 from onegov.fsi.models.course_subscription import subscription_table
 from onegov.search import ORMSearchable
 
-
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from .course import Course
     from .course_notification_template import (
-        CancellationTemplate, CourseNotificationTemplate, InfoTemplate,
-        ReminderTemplate, SubscriptionTemplate
+        CancellationTemplate,
+        CourseNotificationTemplate,
+        InfoTemplate,
+        ReminderTemplate,
+        SubscriptionTemplate
     )
 
 COURSE_EVENT_STATUSES = ('created', 'confirmed', 'canceled', 'planned')
@@ -83,7 +85,7 @@ def es_public(self):
     def title(self):
         return str(self)
 
-    @property
+    @hybrid_property
     def name(self):
         return self.course.name
 
@@ -95,7 +97,7 @@ def lead(self):
             f'{self.presenter_company}'
         )
 
-    @property
+    @hybrid_property
     def description(self):
         return self.course.description
 

src/onegov/fsi/views/search.py

@@ -1,15 +1,29 @@
 from onegov.core.security import Personal
 from onegov.fsi import FsiApp
-from onegov.org.models import Search
+from onegov.org.models import Search, SearchPostgres
 from onegov.org.views.search import search as search_view
+from onegov.org.views.search import search_postgres as search_postgres_view
 from onegov.org.views.search import suggestions as suggestions_view
+from onegov.org.views.search import suggestions_postgres as \
+    suggestions_postgres_view
 
 
 @FsiApp.html(model=Search, template='search.pt', permission=Personal)
 def search(self, request):
     return search_view(self, request)
 
 
+@FsiApp.html(model=SearchPostgres, template='search_postgres.pt',
+             permission=Personal)
+def search_postgres(self, request):
+    return search_postgres_view(self, request)
+
+
 @FsiApp.json(model=Search, name='suggest', permission=Personal)
 def suggestions(self, request):
     return suggestions_view(self, request)
+
+
+@FsiApp.json(model=SearchPostgres, name='suggest', permission=Personal)
+def suggestions_postgres(self, request):
+    return suggestions_postgres_view(self, request)

src/onegov/landsgemeinde/models/agenda.py

@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.core.orm import Base
 from onegov.core.orm.mixins import content_property
 from onegov.core.orm.mixins import ContentMixin
@@ -105,6 +107,18 @@ class AgendaItem(
 
     last_modified = Column(UTCDateTime)
 
+    @hybrid_property
+    def overview(self):  # noqa: F811
+        return self.content['overview'].astext
+
+    @hybrid_property
+    def text(self):  # noqa: F811
+        return self.content['text'].astext
+
+    @hybrid_property
+    def resolution(self):  # noqa: F811
+        return self.content['resolution'].astext
+
     def stamp(self):
         self.last_modified = self.timestamp()
 

src/onegov/landsgemeinde/models/assembly.py

@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.core.orm import Base
 from onegov.core.orm.mixins import content_property
 from onegov.core.orm.mixins import ContentMixin
@@ -94,5 +96,9 @@ def es_suggestion(self):
 
     last_modified = Column(UTCDateTime)
 
+    @hybrid_property
+    def overview(self):  # noqa: F811
+        return self.content['overview'].astext
+
     def stamp(self):
         self.last_modified = self.timestamp()

src/onegov/landsgemeinde/models/votum.py

@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.core.orm import Base
 from onegov.core.orm.mixins import content_property
 from onegov.core.orm.mixins import ContentMixin
@@ -92,6 +94,18 @@ def es_suggestion(self):
         nullable=False
     )
 
+    @hybrid_property
+    def text(self):  # noqa: F811
+        return self.content['text'].astext
+
+    @hybrid_property
+    def motion(self):  # noqa: F811
+        return self.content['motion'].astext
+
+    @hybrid_property
+    def statement_of_reasons(self):  # noqa: F811
+        return self.content['statement_of_reasons'].astext
+
     @property
     def date(self):
         return self.agenda_item.date

src/onegov/landsgemeinde/views/search.py

@@ -7,4 +7,5 @@
 
 @LandsgemeindeApp.html(model=Search, template='search.pt', permission=Public)
 def landsgemeinde_search(self, request):
+    # TODO: switch to postgres search
     return search(self, request, DefaultLayout(self, request))

src/onegov/newsletter/models.py

@@ -95,6 +95,9 @@ def validate_name(self, key: str, name: str) -> str:
         back_populates='newsletters')
 
     @property
+    def search_score(self) -> int:
+        return 6
+
     def open_recipients(self) -> tuple['Recipient', ...]:
         received = select([newsletter_recipients.c.recipient_id]).where(
             newsletter_recipients.c.newsletter_id == self.name)

src/onegov/onboarding/app.py

@@ -2,15 +2,15 @@
 from onegov.file import DepotApp
 from onegov.onboarding.theme import OnboardingTheme
 from onegov.reservation import LibresIntegration
-from onegov.search import ElasticsearchApp
+from onegov.search import SearchApp
 
 
 from typing import Any, TYPE_CHECKING
 if TYPE_CHECKING:
     from collections.abc import Iterator
 
 
-class OnboardingApp(Framework, LibresIntegration, DepotApp, ElasticsearchApp):
+class OnboardingApp(Framework, LibresIntegration, DepotApp, SearchApp):
 
     serve_static_files = True
 

src/onegov/org/app.py

@@ -29,7 +29,7 @@
 from onegov.page import Page, PageCollection
 from onegov.pay import PayApp
 from onegov.reservation import LibresIntegration
-from onegov.search import ElasticsearchApp
+from onegov.search import SearchApp
 from onegov.ticket import TicketCollection
 from onegov.ticket import TicketPermission
 from onegov.user import UserApp
@@ -47,7 +47,7 @@
     from reg.dispatch import _KeyLookup
 
 
-class OrgApp(Framework, LibresIntegration, ElasticsearchApp, MapboxApp,
+class OrgApp(Framework, LibresIntegration, SearchApp, MapboxApp,
              DepotApp, PayApp, FormApp, UserApp, WebsocketsApp):
 
     serve_static_files = True

src/onegov/org/layout.py

@@ -26,7 +26,7 @@
 from onegov.org import _
 from onegov.org import utils
 from onegov.org.exports.base import OrgExport
-from onegov.org.models import ExportCollection, Editor
+from onegov.org.models import ExportCollection, Editor, SearchPostgres
 from onegov.org.models import GeneralFileCollection
 from onegov.org.models import ImageFile
 from onegov.org.models import ImageFileCollection
@@ -290,11 +290,16 @@ def homepage_url(self):
     @cached_property
     def search_url(self):
         """ Returns the url to the search page. """
+        if 'search_postgres' in self.request.path_info:
+            return self.request.link(SearchPostgres(self.request, None, None))
         return self.request.link(Search(self.request, None, None))
 
     @cached_property
     def suggestions_url(self):
         """ Returns the url to the suggestions json view. """
+        if 'search_postgres' in self.request.path_info:
+            return self.request.link(SearchPostgres(self.request, None,
+                                                    None), 'suggest')
         return self.request.link(Search(self.request, None, None), 'suggest')
 
     @cached_property

src/onegov/org/models/__init__.py

@@ -46,7 +46,7 @@
 from onegov.org.models.recipient import ResourceRecipient
 from onegov.org.models.recipient import ResourceRecipientCollection
 from onegov.org.models.resource import DaypassResource
-from onegov.org.models.search import Search
+from onegov.org.models.search import Search, SearchPostgres
 from onegov.org.models.sitecollection import SiteCollection
 from onegov.org.models.swiss_holidays import SwissHolidays
 from onegov.org.models.tan import TAN
@@ -103,6 +103,7 @@
     'ResourceRecipient',
     'ResourceRecipientCollection',
     'Search',
+    'SearchPostgres',
     'SiteCollection',
     'SubmissionMessage',
     'SwissHolidays',

src/onegov/org/models/external_link.py

@@ -1,5 +1,7 @@
 from uuid import uuid4
 
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.core.collection import GenericCollection
 from onegov.core.orm import Base
 from onegov.core.orm.mixins import ContentMixin, \
@@ -48,10 +50,18 @@ class ExternalLink(Base, ContentMixin, TimestampMixin, AccessExtension,
 
     lead = meta_property()
 
+    @property
+    def search_score(self):
+        return 8
+
     @observes('title')
     def title_observer(self, title):
         self.order = normalize_for_url(title)
 
+    @hybrid_property
+    def lead(self):  # noqa: F811
+        return self.meta['lead'].astext
+
 
 class ExternalLinkCollection(GenericCollection):
 

src/onegov/org/models/file.py

@@ -7,6 +7,9 @@
 from dateutil.relativedelta import relativedelta
 from functools import cached_property
 from itertools import chain, groupby
+
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.core.orm import as_selectable
 from onegov.core.orm.mixins import meta_property
 from onegov.file import File, FileSet, FileCollection, FileSetCollection
@@ -153,6 +156,14 @@ def es_suggestions(self):
 
     show_images_on_homepage = meta_property()
 
+    @hybrid_property
+    def lead(self):  # noqa: F811
+        return self.meta['lead'].astext
+
+    @hybrid_property
+    def view(self):  # noqa: F811
+        return self.meta['view'].astext
+
 
 class ImageSetCollection(FileSetCollection):
 

src/onegov/org/models/form.py

@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.form.models import FormDefinition
 from onegov.org.models.extensions import AccessExtension
 from onegov.org.models.extensions import ContactExtension
@@ -21,6 +23,14 @@ class BuiltinFormDefinition(FormDefinition, AccessExtension,
     def extensions(self):
         return tuple(set(super().extensions + ['honeypot']))
 
+    @hybrid_property
+    def lead(self):
+        return self.meta['lead'].astext
+
+    @hybrid_property
+    def text(self):
+        return self.content['text'].astext
+
 
 class CustomFormDefinition(FormDefinition, AccessExtension,
                            ContactExtension, PersonLinkExtension,

src/onegov/org/models/page.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+
 from onegov.core.orm.mixins import (
     content_property, dict_property, meta_property)
 from onegov.file import MultiAssociatedFiles
@@ -21,6 +22,7 @@
 from sedate import replace_timezone
 from sqlalchemy import desc, func, or_, and_
 from sqlalchemy.dialects.postgresql import array, JSON
+from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import undefer, object_session
 from sqlalchemy_utils import observes
 
@@ -42,6 +44,30 @@ class Topic(Page, TraitInfo, SearchableContent, AccessExtension,
     # Show the lead on topics page
     lead_when_child = content_property(default=True)
 
+    @hybrid_property
+    def lead(self):  # noqa: F811
+        return self.content['lead']
+
+    @lead.expression
+    def lead(cls):
+        return cls.content['lead'].astext
+
+    @hybrid_property
+    def text(self):  # noqa: F811
+        return self.content['text']
+
+    @text.expression
+    def text(cls):
+        return cls.content['text'].astext
+
+    @hybrid_property
+    def url(self):  # noqa: F811
+        return self.content['url']
+
+    @url.expression
+    def url(cls):
+        return cls.content['url'].astext
+
     @property
     def es_skip(self):
         return self.meta.get('trait') == 'link'  # do not index links

src/onegov/org/models/search.py

@@ -1,11 +1,17 @@
+from functools import cached_property
+from operator import attrgetter
+
 from elasticsearch_dsl.function import SF
 from elasticsearch_dsl.query import FunctionScore
 from elasticsearch_dsl.query import Match
 from elasticsearch_dsl.query import MatchPhrase
 from elasticsearch_dsl.query import MultiMatch
-from functools import cached_property
+from sqlalchemy import func
+
 from onegov.core.collection import Pagination
+from onegov.core.orm import Base
 from onegov.event.models import Event
+from onegov.search.utils import searchable_sqlalchemy_models
 
 
 class Search(Pagination):
@@ -133,3 +139,149 @@ def suggestions(self):
         return tuple(self.request.app.es_suggestions_by_request(
             self.request, self.query
         ))
+
+
+def locale_mapping(locale):
+    mapping = {'de_CH': 'german', 'fr_CH': 'french', 'it_CH': 'italian',
+               'rm_CH': 'english'}
+    return mapping.get(locale, 'english')
+
+
+class SearchPostgres(Pagination):
+    """
+    Implements searching in postgres db based on the gin index
+    """
+    results_per_page = 10
+    max_query_length = 100
+
+    def __init__(self, request, query, page):
+        self.request = request
+        self.query = query
+        self.page = page  # page index
+
+        self.nbr_of_docs = 0
+        self.nbr_of_results = 0
+
+    @cached_property
+    def available_documents(self):
+        if not self.nbr_of_docs:
+            self.load_batch_results
+        return self.nbr_of_docs
+
+    @cached_property
+    def available_results(self):
+        if not self.nbr_of_results:
+            self.load_batch_results
+        return self.nbr_of_results
+
+    @property
+    def q(self):
+        return self.query
+
+    def __eq__(self, other):
+        return self.page == other.page and self.query == other.query
+
+    def subset(self):
+        return self.batch
+
+    @property
+    def page_index(self):
+        return self.page
+
+    def page_by_index(self, index):
+        return SearchPostgres(self.request, self.query, index)
+
+    @cached_property
+    def batch(self):
+        if not self.query:
+            return None
+
+        if self.query.startswith('#'):
+            results = self.hashtag_search()
+        else:
+            results = self.generic_search()
+
+        return results[self.offset:self.offset + self.batch_size]
+
+    @cached_property
+    def load_batch_results(self):
+        """Load search results and sort events by latest occurrence.
+
+        This methods is a wrapper around `batch.load()`, which returns the
+        actual search results form the query. """
+
+        batch = self.batch
+        events = []
+        non_events = []
+        for search_result in batch:
+            if isinstance(search_result, Event):
+                events.append(search_result)
+            else:
+                non_events.append(search_result)
+        if not events:
+            return batch
+        sorted_events = sorted(events, key=lambda e: e.latest_occurrence.start)
+        return sorted_events + non_events
+
+    def generic_search(self):
+        doc_count = 0
+        results = []
+
+        language = locale_mapping(self.request.locale)
+        for model in searchable_sqlalchemy_models(Base):
+            if model.es_public or self.request.is_logged_in:
+                query = self.request.session.query(model)
+                doc_count += query.count()
+                query = query.filter(
+                    model.fts_idx.op('@@')(func.websearch_to_tsquery(
+                        language, self.query))
+                )
+                query = query.order_by(func.ts_rank_cd(
+                    model.fts_idx, func.websearch_to_tsquery(language,
+                                                             self.query)))
+                results.extend(query.all())
+
+        self.nbr_of_docs = doc_count
+        self.nbr_of_results = len(results)
+        results.sort(key=attrgetter('search_score'), reverse=False)
+        return results
+
+    def hashtag_search(self):
+        q = self.query.lstrip('#')
+        results = []
+
+        for model in searchable_sqlalchemy_models(Base):
+            # skip certain tables for hashtag search for better performance
+            if model.__tablename__ not in ['attendees', 'files', 'people',
+                                           'tickets', 'users']:
+                if model.es_public or self.request.is_logged_in:
+                    for doc in self.request.session.query(model).all():
+                        if doc.es_tags and q in doc.es_tags:
+                            results.append(doc)
+
+        self.nbr_of_results = len(results)
+        results.sort(key=attrgetter('search_score'), reverse=False)
+        return results
+
+    def feeling_lucky(self):
+        if self.batch:
+            first_entry = self.batch[0].load()
+
+            # XXX the default view to the event should be doing the redirect
+            if first_entry.__tablename__ == 'events':
+                return self.request.link(first_entry, 'latest')
+            else:
+                return self.request.link(first_entry)
+
+    @cached_property
+    def subset_count(self):
+        return self.available_results
+
+    def suggestions(self):
+        suggestions = list()
+
+        for element in self.generic_search():
+            suggest = getattr(element, 'es_suggestion', [])
+            suggestions.append(suggest)
+
+        return tuple(suggestions[:15])

src/onegov/org/path.py

@@ -55,6 +55,7 @@
 from onegov.org.models import ResourceRecipient
 from onegov.org.models import ResourceRecipientCollection
 from onegov.org.models import Search
+from onegov.org.models import SearchPostgres
 from onegov.org.models import SiteCollection
 from onegov.org.models import TicketNote
 from onegov.org.models import Topic
@@ -506,6 +507,11 @@ def get_search(request, q='', page=0):
     return Search(request, q, page)
 
 
+@OrgApp.path(model=SearchPostgres, path='/search_postgres')
+def get_postgres_search(request, q='', page=0):
+    return SearchPostgres(request, q, page)
+
+
 @OrgApp.path(model=AtoZPages, path='/a-z')
 def get_a_to_z(request):
     return AtoZPages(request)

src/onegov/org/templates/search_postgres.pt

@@ -0,0 +1,69 @@
+<div metal:use-macro="layout.base" i18n:domain="onegov.org">
+    <tal:b metal:fill-slot="title">
+        ${title}
+    </tal:b>
+    <tal:b metal:fill-slot="content">
+        <div class="grid-x" tal:condition="not:connection">
+            <div class="cell small-12 medium-8" i18n:translate>
+                Postgres Searching is currently unavailable due to technical
+                difficulties.
+                Please excuse the inconvenience and try again later.
+            </div>
+        </div>
+
+        <div class="grid-x" tal:condition="connection">
+            <div class="cell small-12 medium-8">
+                <form class="searchbox" action="${request.link(model).split('?')[0]}" method="GET" data-typeahead="on" data-typeahead-source="${layout.suggestions_url}" data-typeahead-target="${layout.search_url}">
+                <label>${searchlabel}</label>
+                <div class="input-group">
+                    <input class="input-group-field" id="search" data-typeahead-subject type="search" name="q" value="${model.query}" autocomplete="off" autocorrect="off" required autofocus />
+                    <div class="input-group-button">
+                        <button type="submit" class="button" aria-label="Search" i18n:attributes="aria-label">
+                            <i class="fa fa-fw fa-search"></i>
+                        </button>
+                    </div>
+                </div>
+                <div class="grid-x typeahead">
+                    <div class="cell small-12" data-typeahead-container></div>
+                </div>
+                </form>
+            </div>
+        </div>
+
+        <tal:b condition="connection and model.query">
+            <h2 i18n:translate>${resultslabel}</h2>
+            <div class="grid-x">
+                <div class="cell medium-8 small-12">
+                    <p tal:condition="not: model.batch" i18n:translate=""> Your postgres search returned no results.</p>
+
+                    <ul class="more-list">
+                        <tal:b repeat="result model.load_batch_results">
+                            <li class="search-result-${result.es_type_name}" tal:condition="result">
+                                    <tal:b
+                                        define="macro 'search_result_{}'.format(result.__tablename__); tenant_macro 'search_result_{}_{}'.format(result.__tablename__, request.app.application_id.split('/')[-1])"
+                                        metal:use-macro="layout.macros[tenant_macro]|layout.macros[macro]|layout.macros['search_result_default']"
+                                    />
+
+                                    <tal:b condition="request.is_manager and result.explanation|nothing">
+                                        <ul class="result-explanation">
+                                            <li>Score: ${result.explanation['score']}</li>
+                                            <tal:b repeat="(title, key) [('TF', 'term-frequency'), ('IDF', 'inverse-document-frequency'), ('Field Norm', 'field-norm')]">
+                                                <li title="${result.explanation[key]['description']}" tal:condition="result.explanation[key]">
+                                                    ${title}: ${result.explanation[key]['value']}
+                                                </li>
+                                            </tal:b>
+                                        </ul>
+                                    </tal:b>
+                                </li>
+                        </tal:b>
+                    </ul>
+
+
+                        <div metal:use-macro="layout.macros['pagination']" tal:define="
+                        collection model; current_page model; pagination_centered True" />
+
+                </div>
+            </div>
+        </tal:b>
+    </tal:b>
+</div>

src/onegov/org/views/search.py

@@ -4,14 +4,13 @@
 from onegov.org import _, OrgApp
 from onegov.org.elements import Link
 from onegov.org.layout import DefaultLayout
-from onegov.org.models import Search
+from onegov.org.models import Search, SearchPostgres
 from onegov.search import SearchOfflineError
 from webob import exc
 
 
 @OrgApp.html(model=Search, template='search.pt', permission=Public)
 def search(self, request, layout=None):
-
     layout = layout or DefaultLayout(self, request)
     layout.breadcrumbs.append(Link(_("Search"), '#'))
 
@@ -46,9 +45,56 @@ def search(self, request, layout=None):
     }
 
 
+@OrgApp.html(model=SearchPostgres, template='search_postgres.pt',
+             permission=Public)
+def search_postgres(self, request, layout=None):
+    layout = layout or DefaultLayout(self, request)
+    layout.breadcrumbs.append(Link(_("Search"), '#'))
+
+    try:
+        searchlabel = _("Search through ${count} indexed documents", mapping={
+            'count': self.available_documents
+        })
+        resultslabel = _("${count} Results", mapping={
+            'count': self.available_results
+        })
+    except SearchOfflineError:
+        return {
+            'title': _("Search Unavailable"),
+            'layout': layout,
+            'connection': False
+        }
+
+    if 'lucky' in request.GET:
+        url = self.feeling_lucky()
+
+        if url:
+            return morepath.redirect(url)
+
+    return {
+        # TODO switch back to 'Search' once es is gone
+        # 'title': _("Search"),
+        'title': _("Org Search Postgres"),
+        'model': self,
+        'layout': layout,
+        'hide_search_header': True,
+        'searchlabel': searchlabel,
+        'resultslabel': resultslabel,
+        'connection': True
+    }
+
+
 @OrgApp.json(model=Search, name='suggest', permission=Public)
 def suggestions(self, request):
     try:
         return tuple(self.suggestions())
     except SearchOfflineError as exception:
         raise exc.HTTPNotFound() from exception
+
+
+@OrgApp.json(model=SearchPostgres, name='suggest', permission=Public)
+def suggestions_postgres(self, request):
+    try:
+        return tuple(self.suggestions())
+    except SearchOfflineError as exception:
+        raise exc.HTTPNotFound() from exception

src/onegov/page/model.py

@@ -25,6 +25,10 @@ class Page(AdjacencyList, ContentMixin, TimestampMixin, UTCPublicationMixin):
 
     __tablename__ = 'pages'
 
+    @property
+    def search_score(self) -> int:
+        return 2
+
     if TYPE_CHECKING:
         # we override these relationships to be more specific
         parent: relationship['Page']

src/onegov/people/models/membership.py

@@ -88,6 +88,10 @@ class AgencyMembership(Base, ContentMixin, TimestampMixin, ORMSearchable,
     #: when the membership started
     since = Column(Text, nullable=True)
 
+    @property
+    def search_score(self):
+        return 3
+
     @property
     def siblings_by_agency(self):
         """ Returns a query that includes all siblings by agency, including

src/onegov/people/models/person.py

@@ -1,11 +1,12 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.core.orm import Base
 from onegov.core.orm.mixins import ContentMixin
 from onegov.core.orm.mixins import TimestampMixin
 from onegov.core.orm.mixins import UTCPublicationMixin
 from onegov.core.orm.types import UUID
 from onegov.search import ORMSearchable
-from sqlalchemy import Column
-from sqlalchemy import Text
+from sqlalchemy import Column, Text
 from uuid import uuid4
 from vobject import vCard
 from vobject.vcard import Address
@@ -38,9 +39,9 @@ class Person(Base, ContentMixin, TimestampMixin, ORMSearchable,
 
     @property
     def es_suggestion(self):
-        return (self.title, f"{self.first_name} {self.last_name}")
+        return self.title
 
-    @property
+    @hybrid_property
     def title(self):
         """ Returns the Estern-ordered name. """
 
@@ -129,6 +130,10 @@ def spoken_title(self):
     #: some remarks about the person
     notes = Column(Text, nullable=True)
 
+    @property
+    def search_score(self):
+        return 3
+
     def vcard_object(self, exclude=None, include_memberships=True):
         """ Returns the person as vCard (3.0) object.
 

src/onegov/search/__init__.py

@@ -4,11 +4,11 @@
 
 from onegov.search.mixins import Searchable, ORMSearchable, SearchableContent
 from onegov.search.dsl import Search
-from onegov.search.integration import ElasticsearchApp
+from onegov.search.integration import SearchApp
 from onegov.search.errors import SearchOfflineError
 
 __all__ = [
-    'ElasticsearchApp',
+    'SearchApp',
     'ORMSearchable',
     'Search',
     'Searchable',

src/onegov/search/cli.py

@@ -5,26 +5,33 @@
 from onegov.core.cli import command_group, pass_group_context
 from sedate import utcnow
 
-
 cli = command_group()
 
 
 @cli.command(context_settings={'default_selector': '*'})
 @click.option('--fail', is_flag=True, default=False, help='Fail on errors')
 @pass_group_context
 def reindex(group_context, fail):
-    """ Reindexes all objects in the elasticsearch database. """
+    """ Reindexes all objects in the postgresql database. """
 
     def run_reindex(request, app):
-        if not hasattr(request.app, 'es_client'):
-            return
+        """
+        Looping over all models in project deleting all full text search (
+        fts) indexes in postgresql and re-creating them
 
+        :param request: request
+        :param app: application context
+        """
         title = f"Reindexing {request.app.application_id}"
         print(click.style(title, underline=True))
 
         start = utcnow()
-        request.app.es_perform_reindex(fail)
+        app.psql_perform_reindex(request)
+        print(f"- psql indexing took {utcnow() - start}")
 
-        print(f"took {utcnow() - start}")
+        # TODO: remove es indexing once es is gone
+        start = utcnow()
+        request.app.es_perform_reindex(fail)
+        print(f"- es indexing took {utcnow() - start}")
 
     return run_reindex

src/onegov/search/integration.py

@@ -10,6 +10,8 @@
 from elasticsearch import TransportError
 from elasticsearch.connection import create_ssl_context
 from more.transaction.main import transaction_tween_factory
+
+from onegov.core.orm import Base
 from onegov.search import Search, log
 from onegov.search.errors import SearchOfflineError
 from onegov.search.indexer import Indexer
@@ -90,8 +92,9 @@ def is_5xx_error(error):
     return error.status_code and str(error.status_code).startswith('5')
 
 
-class ElasticsearchApp(morepath.App):
-    """ Provides elasticsearch integration for
+# TODO: remove all es specific things ones es is gone
+class SearchApp(morepath.App):
+    """ Provides elasticsearch and postgres integration for
     :class:`onegov.core.framework.Framework` based applications.
 
     The application must be connected to a database.
@@ -326,6 +329,7 @@ def es_suggestions_by_request(self, request, query, types='*',
         else:
             languages = '*'
 
+        print(f'es_suggestion_by_request language: {languages}')
         return self.es_suggestions(
             query,
             languages=languages,
@@ -394,8 +398,20 @@ def reindex_model(model):
 
         self.es_indexer.bulk_process()
 
+    def psql_perform_reindex(self, request):
+        """ Re-indexes all `searchable' models in postgresql db ensuring
+        each table will be indexed only once.
+
+        """
+        done = []
+
+        for model in searchable_sqlalchemy_models(Base):
+            if model.__tablename__ not in done:
+                model.reindex(request, model)
+                done.append(model.__tablename__)
+
 
-@ElasticsearchApp.tween_factory(over=transaction_tween_factory)
+@SearchApp.tween_factory(over=transaction_tween_factory)
 def process_indexer_tween_factory(app, handler):
     def process_indexer_tween(request):
 

src/onegov/search/mixins.py

@@ -1,6 +1,15 @@
-from onegov.search.utils import classproperty
+from sqlalchemy import Column, func, Computed  # type:ignore[attr-defined]
+from sqlalchemy.dialects.postgresql import TSVECTOR
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import deferred
+
+from onegov.core.upgrade import UpgradeContext
+from onegov.search.utils import classproperty, \
+    get_fts_index_localized_languages, get_fts_index_basic_languages
 from onegov.search.utils import extract_hashtags
 
+from typing import Any, TYPE_CHECKING
+
 
 class Searchable:
     """ Defines the interface required for an object to be searchable.
@@ -38,6 +47,18 @@ def es_type_name(self):
     identity is a completely different model.
 
     """
+    TEXT_SEARCH_COLUMN_NAME = 'fts_idx'
+
+    if TYPE_CHECKING:
+        fts_idx: 'Column[dict[str, Any]]'
+
+    # column for full text search index
+    @declared_attr  # type:ignore[no-redef]
+    def fts_idx(cls) -> 'Column[dict[str, Any]]':
+        col_name = Searchable.TEXT_SEARCH_COLUMN_NAME
+        if hasattr(cls, '__table__') and hasattr(cls.__table__.c, col_name):
+            return deferred(cls.__table__.c.fts_idx)
+        return deferred(Column(col_name, TSVECTOR))
 
     @classproperty
     def es_properties(self):
@@ -134,7 +155,106 @@ def es_tags(self):
         """ Returns a list of tags associated with this content. """
         return None
 
+    @property
+    def search_score(self):
+        """
+        the lower the score the higher the class type will be shown in search
+        results. Default is 10 (lowest)
+        """
+        return 10
+
+    @staticmethod
+    def psql_tsvector_expression(model):
+        """
+        Provides the tsvector expression for postgres for the defined
+        model. Depending on the model columns and properties are used for full
+        text search index.
+
+        :return: tsvector expression
+        """
+        objects = [getattr(model, p) for p in model.es_properties if
+                   not p.startswith('es_')]
+        return Searchable.create_tsvector_expression(*objects)
+
+    @staticmethod
+    def reindex(request, model):
+        """
+        Re-indexes the table by dropping and adding the full text search
+        column.
+        """
+        Searchable.drop_fts_column(request, model)
+        Searchable.add_fts_column(request, model)
+
+    @staticmethod
+    def drop_fts_column(request, model):
+        """
+        Drops the full text search column
+
+        :param request: request object
+        :param model: model to drop the index from
+        :return: None
+        """
+
+        col_name = Searchable.TEXT_SEARCH_COLUMN_NAME
+        context = UpgradeContext(request)
+
+        if context.has_column(model.__tablename__, col_name):
+            context.operations.drop_column(model.__tablename__, col_name)
+
+    @staticmethod
+    def add_fts_column(request, model):
+
+        """
+        This function is used for re-indexing and as migration step moving to
+        postgresql full text search (fts), OGC-508.
+
+        It adds a separate column for the tsvector to `schema`.`table`
+        creating a multilingual gin index on the columns/data defined per
+        model.
+
+        :param request: request object
+        :param model: model to add the index
+        :return: None
+        """
 
+        col_name = Searchable.TEXT_SEARCH_COLUMN_NAME
+        context = UpgradeContext(request)
+        if not context.has_column(model.__tablename__, col_name):
+            tsvector_expression = None
+            for prop_name, type_info in model.es_properties.items():
+                if not prop_name.startswith('es_'):
+                    prop_type = type_info.get('type', None)
+                    prop = getattr(model, prop_name)
+                    languages = get_fts_index_basic_languages()
+
+                    if prop_type in ['localized', 'localized_html']:
+                        # only for 'localized' properties we create the
+                        # index localized
+                        languages.extend(get_fts_index_localized_languages())
+
+                    for language in languages:
+                        expr = func.to_tsvector(language,
+                                                func.coalesce(prop, ''))
+
+                        if tsvector_expression is None:
+                            tsvector_expression = expr
+                        else:
+                            tsvector_expression = tsvector_expression.concat(
+                                expr)
+
+            context.operations.add_column(
+                model.__tablename__,
+                Column(col_name,
+                       TSVECTOR,
+                       Computed(
+                           tsvector_expression,
+                           persisted=True),
+                       )
+            )
+            context.operations.execute("COMMIT")
+
+
+# TODO: rename prefix 'es' to 'ts' for text search
 class ORMSearchable(Searchable):
     """ Extends the default :class:`Searchable` class with sensible defaults
     for SQLAlchemy orm models.
@@ -154,6 +274,7 @@ def es_last_change(self):
         return getattr(self, 'last_change', None)
 
 
+# TODO: rename prefix 'es' to 'ts' for text search
 class SearchableContent(ORMSearchable):
     """ Adds search to all classes using the core's content mixin:
     :class:`onegov.core.orm.mixins.content.ContentMixin`

src/onegov/search/utils.py

@@ -8,7 +8,6 @@
 from langdetect.utils.lang_profile import LangProfile
 from onegov.core.orm import find_models
 
-
 # XXX this is doubly defined in onegov.org.utils, maybe move to a common
 # regex module in in onegov.core
 HASHTAG = re.compile(r'#\w{3,}')
@@ -30,6 +29,16 @@ def searchable_sqlalchemy_models(base):
 _invalid_index_characters = re.compile(r'[\\/?"<>|\s,A-Z:]+')
 
 
+def get_fts_index_languages():
+    """ Define index creation languages for full text search as we have a
+    limited set of used languages.
+
+    NOTE: 'simple' is used for tag, label or phrase searches
+
+    """
+    return ['simple', 'german', 'french', 'italian', 'english']
+
+
 def is_valid_index_name(name):
     """ Checks if the given name is a valid elasticsearch index name.
     Elasticsearch does it's own checks, but we can do it earlier and we are

src/onegov/ticket/model.py

@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.core.orm import Base
 from onegov.core.orm.mixins import TimestampMixin
 from onegov.core.orm.types import JSON, UUID
@@ -105,6 +107,10 @@ class Ticket(Base, TimestampMixin, ORMSearchable):
     #: true if the notifications for this ticket should be muted
     muted: 'Column[bool]' = Column(Boolean, nullable=False, default=False)
 
+    @property
+    def search_score(self) -> int:
+        return 6
+
     if TYPE_CHECKING:
         created: Column[datetime]
     else:
@@ -142,6 +148,7 @@ def created(cls) -> 'Column[datetime]':
 
     # limit the search to the ticket number -> the rest can be found
     es_public = False
+
     es_properties = {
         'number': {'type': 'text'},
         'title': {'type': 'text'},
@@ -152,7 +159,7 @@ def created(cls) -> 'Column[datetime]':
         'extra_localized_text': {'type': 'localized'}
     }
 
-    @property
+    @hybrid_property
     def extra_localized_text(self) -> str | None:
         """ Maybe used by child-classes to return localized extra data that
         should be indexed as well.
@@ -167,14 +174,14 @@ def es_suggestion(self) -> list[str]:
             self.number.replace('-', '')
         ]
 
-    @property
+    @hybrid_property
     def ticket_email(self) -> str | None:
         if self.handler.deleted:
             return self.snapshot.get('email')
         else:
             return self.handler.email
 
-    @property
+    @hybrid_property
     def ticket_data(self) -> 'Sequence[str] | None':
         if self.handler.deleted:
             return self.snapshot.get('summary')

src/onegov/town6/templates/search_postgres.pt

@@ -0,0 +1,70 @@
+<div metal:use-macro="layout.base" i18n:domain="onegov.town6">
+    <tal:b metal:fill-slot="title">
+        ${title}
+    </tal:b>
+    <tal:b metal:fill-slot="content">
+        <div class="grid-x" tal:condition="not:connection">
+            <div class="cell small-12 medium-8" i18n:translate>
+                Postgres Searching is currently unavailable due to technical
+                difficulties.
+                Please excuse the inconvenience and try again later.
+            </div>
+        </div>
+
+        <div class="grid-x" tal:condition="connection">
+            <div class="cell small-12 medium-8">
+                <form class="searchbox" action="${request.link(model).split('?')[0]}" method="GET" data-typeahead="on" data-typeahead-source="${layout.suggestions_url}" data-typeahead-target="${layout.search_url}">
+                <label>${searchlabel}</label>
+                <div class="input-group">
+                    <input class="input-group-field" id="search" data-typeahead-subject type="search" name="q" value="${model.query}" autocomplete="off" autocorrect="off" required autofocus />
+                    <div class="input-group-button">
+                        <button type="submit" class="button" aria-label="Search" i18n:attributes="aria-label">
+                            <i class="fa fa-fw fa-search"></i>
+                        </button>
+                    </div>
+                </div>
+                <div class="grid-x typeahead">
+                    <div class="cell small-12" data-typeahead-container></div>
+                </div>
+                </form>
+            </div>
+        </div>
+
+        <tal:b condition="connection and model.query">
+            <h2 i18n:translate>${resultslabel}</h2>
+            <div class="grid-x">
+                <div class="cell medium-8 small-12">
+<!--?                    TODO: remove 'postgres' from line below once no es anymore -->
+                    <p tal:condition="not: model.batch" i18n:translate=""> Your postgres search returned no results.</p>
+
+                    <ul class="more-list">
+                        <tal:b repeat="result model.load_batch_results">
+                            <li class="search-result-${result.es_type_name}" tal:condition="result">
+                                    <tal:b
+                                        define="macro 'search_result_{}'.format(result.__tablename__); tenant_macro 'search_result_{}_{}'.format(result.__tablename__, request.app.application_id.split('/')[-1])"
+                                        metal:use-macro="layout.macros[tenant_macro]|layout.macros[macro]|layout.macros['search_result_default']"
+                                    />
+
+                                    <tal:b condition="request.is_manager and result.explanation|nothing">
+                                        <ul class="result-explanation">
+                                            <li>Score: ${result.explanation['score']}</li>
+                                            <tal:b repeat="(title, key) [('TF', 'term-frequency'), ('IDF', 'inverse-document-frequency'), ('Field Norm', 'field-norm')]">
+                                                <li title="${result.explanation[key]['description']}" tal:condition="result.explanation[key]">
+                                                    ${title}: ${result.explanation[key]['value']}
+                                                </li>
+                                            </tal:b>
+                                        </ul>
+                                    </tal:b>
+                                </li>
+                        </tal:b>
+                    </ul>
+
+
+                        <div metal:use-macro="layout.macros['pagination']" tal:define="
+                        collection model; current_page model; pagination_centered True" />
+
+                </div>
+            </div>
+        </tal:b>
+    </tal:b>
+</div>

src/onegov/town6/views/search.py

@@ -1,10 +1,16 @@
 from onegov.core.security import Public
-from onegov.org.views.search import search
+from onegov.org.views.search import search, search_postgres
 from onegov.town6 import TownApp
-from onegov.org.models import Search
+from onegov.org.models import Search, SearchPostgres
 from onegov.town6.layout import DefaultLayout
 
 
 @TownApp.html(model=Search, template='search.pt', permission=Public)
 def town_search(self, request):
     return search(self, request, DefaultLayout(self, request))
+
+
+@TownApp.html(model=SearchPostgres, template='search_postgres.pt',
+              permission=Public)
+def town_search_postgres(self, request):
+    return search_postgres(self, request, DefaultLayout(self, request))

src/onegov/translator_directory/models/translator.py

@@ -28,6 +28,7 @@
     from .language import Language
 
 
+# TODO rename to ts (text search)
 class ESMixin(ORMSearchable):
 
     es_properties = {
@@ -182,6 +183,10 @@ class Translator(Base, TimestampMixin, AssociatedFiles, ContentMixin,
     expertise_professional_guilds_other: 'dict_property[Sequence[str]]'
     expertise_professional_guilds_other = meta_property(default=tuple)
 
+    @property
+    def search_score(self):
+        return 4
+
     @property
     def expertise_professional_guilds_all(self):
         return (

src/onegov/user/models/user.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+
 from onegov.core.crypto import hash_password, verify_password
 from onegov.core.orm import Base
 from onegov.core.orm.mixins import data_property, dict_property, TimestampMixin
@@ -9,8 +10,7 @@
 from onegov.core.utils import yubikey_otp_to_serial
 from onegov.search import ORMSearchable
 from onegov.user.models.group import UserGroup
-from sqlalchemy import Boolean, Column, Index, Text, func, ForeignKey
-from sqlalchemy import UniqueConstraint
+from sqlalchemy import Boolean, Column, Text, func, ForeignKey
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import backref, deferred, relationship
 from uuid import uuid4, UUID as UUIDType
@@ -57,7 +57,7 @@ class User(Base, TimestampMixin, ORMSearchable):
     def es_suggestion(self) -> tuple[str, str]:
         return (self.realname or self.username, self.username)
 
-    @property
+    @hybrid_property
     def userprofile(self) -> list[str]:
         if not self.data:
             return []
@@ -150,10 +150,9 @@ def userprofile(self) -> list[str]:
     signup_token: 'Column[str | None]' = Column(
         Text, nullable=True, default=None)
 
-    __table_args__ = (
-        Index('lowercase_username', func.lower(username), unique=True),
-        UniqueConstraint('source', 'source_id', name='unique_source_id'),
-    )
+    @property
+    def search_score(self) -> int:
+        return 5
 
     if TYPE_CHECKING:
         # HACK: This probably won't be necessary in SQLAlchemy 2.0, but

tests/onegov/search/test_indexer.py

@@ -2,6 +2,11 @@
 import pytest
 
 from datetime import datetime
+
+from onegov.core.orm import Base
+from onegov.directory import DirectoryEntry
+from onegov.org.models import Topic
+from onegov.people import Agency
 from onegov.search import Searchable, SearchOfflineError, utils
 from onegov.search.indexer import parse_index_name
 from onegov.search.indexer import (
@@ -14,6 +19,10 @@
 from queue import Queue
 from unittest.mock import Mock
 
+from onegov.search.utils import searchable_sqlalchemy_models
+from onegov.ticket import Ticket
+from onegov.user import User
+
 
 def test_index_manager_assertions(es_client):
 
@@ -717,3 +726,88 @@ def test_elasticsearch_outage(es_client, es_url):
     indexer.es_client.indices.refresh(index='_all')
     assert indexer.es_client\
         .search(index='_all')['hits']['total']['value'] == 2
+
+
+def test_psql_tsvector_string():
+    assert Searchable.create_tsvector_string(('col_lower')) == \
+        "'func.coalesce(col_lower, '')'"
+
+    # FIXME: implement lower
+    # assert Searchable.create_tsvector_string(['Col_Higher']) == \
+    #        'coalesce("\'col_higher\'", \'\')'
+
+    assert Searchable.create_tsvector_string('col_a', 'col_b') == \
+        "'func.coalesce(col_a, '')' || ' ' || 'func.coalesce(col_b, '')'"
+
+    assert Searchable.create_tsvector_string('a', 'b', 'c') == \
+        "'func.coalesce(a, '')' || ' ' || 'func.coalesce(b, '')' || ' ' || " \
+        "'func.coalesce(c, '')'"
+
+    cols = ['col_a', 'col_b']
+    assert Searchable.create_tsvector_string(*cols) == \
+           "'func.coalesce(col_a, '')' || ' ' || 'func.coalesce(col_b, '')'"
+
+
+def test_multi_language_tsvector_expression(monkeypatch):
+    tsvector_string = "'func.coalesce(my_col, '')'"
+    x = Searchable.multi_language_tsvector_expression(tsvector_string)
+    assert x == "to_tsvector('simple', 'func.coalesce(my_col, '')') || " \
+                "to_tsvector('german', 'func.coalesce(my_col, '')') || " \
+                "to_tsvector('french', 'func.coalesce(my_col, '')') || " \
+                "to_tsvector('italian', 'func.coalesce(my_col, '')') || " \
+                "to_tsvector('english', 'func.coalesce(my_col, '')')"
+
+    def fake():
+        return ['simple']
+    tsvector_string = "'func.coalesce(group, '')'"
+    monkeypatch.setattr(utils, 'get_fts_index_languages', fake)
+    assert Searchable.multi_language_tsvector_expression(
+        tsvector_string) == "to_tsvector('simple', 'func.coalesce(group, '')')"
+
+
+def test_psql_tsvector_string_generation_models():
+    count = 0
+
+    for model in searchable_sqlalchemy_models(Base):
+        print(f'model {model}..')
+        tsvector = model.psql_tsvector_string(model)
+        for p in getattr(model, 'es_properties', []):
+            if p in model.__dict__ and not p.startswith('_es'):
+                # verify all properties are reflected in the tsvector
+                assert p in tsvector
+
+        # random sample
+        if model == Agency:
+            count += 1
+            assert tsvector == "'func.coalesce(title, '')' || ' ' || " \
+                               "'func.coalesce(description, '')' || ' ' || " \
+                               "'func.coalesce(portrait, '')'"
+        elif model == User:
+            count += 1
+            assert tsvector == "'func.coalesce(username, '')' || ' ' || " \
+                               "'func.coalesce(realname, '')' || ' ' || " \
+                               "'func.coalesce(userprofile, '')'"
+        elif model == DirectoryEntry:
+            count += 1
+            assert tsvector == "'func.coalesce(keywords, '')' || ' ' || " \
+                               "'func.coalesce(title, '')' || ' ' || " \
+                               "'func.coalesce(lead, '')' || ' ' || " \
+                               "'func.coalesce(directory_id, '')' || ' ' || " \
+                               "'func.coalesce(text, '')'"
+        elif model == Ticket:
+            count += 1
+            assert tsvector == "'func.coalesce(number, '')' || ' ' || " \
+                               "'func.coalesce(title, '')' || ' ' || " \
+                               "'func.coalesce(subtitle, '')' || ' ' || " \
+                               "'func.coalesce(group, '')' || ' ' || " \
+                               "'func.coalesce(ticket_email, '')' || ' ' || " \
+                               "'func.coalesce(ticket_data, '')' || ' ' || " \
+                               "'func.coalesce(extra_localized_text, '')'"
+        elif model == Topic:
+            count += 1
+            assert tsvector == "'func.coalesce(title, '')' || ' ' || " \
+                               "'func.coalesce(lead, '')' || ' ' || " \
+                               "'func.coalesce(text, '')'"
+
+    # verify if statements reached and tested
+    assert count == 5

tests/onegov/search/test_integration.py

@@ -5,10 +5,11 @@
 from datetime import timedelta
 from elasticsearch_dsl.function import SF
 from elasticsearch_dsl.query import MatchPhrase, FunctionScore
+
 from onegov.core import Framework
 from onegov.core.orm.mixins import TimestampMixin
 from onegov.core.utils import scan_morepath_modules
-from onegov.search import ElasticsearchApp, ORMSearchable
+from onegov.search import SearchApp, ORMSearchable
 from sqlalchemy import Boolean, Column, Integer, Text
 from sqlalchemy.ext.declarative import declarative_base
 from webtest import TestApp as Client
@@ -17,7 +18,7 @@
 
 def test_app_integration(es_url):
 
-    class App(Framework, ElasticsearchApp):
+    class App(Framework, SearchApp):
         pass
 
     app = App()
@@ -33,7 +34,7 @@ class App(Framework, ElasticsearchApp):
 
 def test_search_query(es_url, postgres_dsn):
 
-    class App(Framework, ElasticsearchApp):
+    class App(Framework, SearchApp):
         pass
 
     Base = declarative_base()
@@ -46,12 +47,20 @@ class Document(Base, ORMSearchable):
         body = Column(Text, nullable=True)
         public = Column(Boolean, nullable=False)
         language = Column(Text, nullable=False)
+        # fts_idx = Column(TSVECTOR, Computed('', persisted=True))
+        # __table_args__ = (
+        #     Index('fts_idx', fts_idx, postgresql_using='gin'),
+        # )
 
         es_properties = {
             'title': {'type': 'localized'},
             'body': {'type': 'localized'}
         }
 
+        # @staticmethod
+        # def psql_tsvector_string():
+        #     return Searchable.create_tsvector_string('title', 'body')
+
         @property
         def es_suggestion(self):
             return self.title
@@ -148,10 +157,17 @@ def es_language(self):
     assert document.title == "Öffentlich"
     assert document.public
 
+    ##################
+    # postgresql tests
+    # app.psql_perform_reindex()
+
+    # results = app.psql_search('')
+    # assert results
+
 
 def test_orm_integration(es_url, postgres_dsn, redis_url):
 
-    class App(Framework, ElasticsearchApp):
+    class App(Framework, SearchApp):
         pass
 
     Base = declarative_base()
@@ -294,7 +310,7 @@ def view_delete_document(self, request):
 
 def test_alternate_id_property(es_url, postgres_dsn):
 
-    class App(Framework, ElasticsearchApp):
+    class App(Framework, SearchApp):
         pass
 
     Base = declarative_base()
@@ -356,7 +372,7 @@ def es_suggestion(self):
 
 def test_orm_polymorphic(es_url, postgres_dsn):
 
-    class App(Framework, ElasticsearchApp):
+    class App(Framework, SearchApp):
         pass
 
     Base = declarative_base()
@@ -442,7 +458,7 @@ def update():
 
 def test_orm_polymorphic_sublcass_only(es_url, postgres_dsn):
 
-    class App(Framework, ElasticsearchApp):
+    class App(Framework, SearchApp):
         pass
 
     Base = declarative_base()
@@ -498,7 +514,7 @@ def es_suggestion(self):
 
 def test_suggestions(es_url, postgres_dsn):
 
-    class App(Framework, ElasticsearchApp):
+    class App(Framework, SearchApp):
         pass
 
     Base = declarative_base()
@@ -617,7 +633,7 @@ def es_suggestion(self):
 
 def test_language_detection(es_url, postgres_dsn):
 
-    class App(Framework, ElasticsearchApp):
+    class App(Framework, SearchApp):
         pass
 
     Base = declarative_base()
@@ -671,7 +687,7 @@ class Document(Base, ORMSearchable):
 
 
 def test_language_update(es_url, postgres_dsn):
-    class App(Framework, ElasticsearchApp):
+    class App(Framework, SearchApp):
         pass
 
     Base = declarative_base()
@@ -725,7 +741,7 @@ class Document(Base, ORMSearchable):
 
 def test_date_decay(es_url, postgres_dsn):
 
-    class App(Framework, ElasticsearchApp):
+    class App(Framework, SearchApp):
         pass
 
     Base = declarative_base()

tests/onegov/search/test_utils.py

@@ -1,5 +1,6 @@
-from onegov.search import ORMSearchable, Searchable
+from onegov.search import ORMSearchable
 from onegov.search import utils
+from onegov.search.mixins import Searchable
 from sqlalchemy import Column, Integer, Text
 from sqlalchemy.ext.declarative import declarative_base
 
@@ -101,3 +102,13 @@ class News(Page):
         es_type_name = 'news'
 
     assert utils.related_types(Page) == {'news', 'topic'}
+
+
+def test_create_tsvector_string():
+    assert Searchable.create_tsvector_string('username') == \
+           "coalesce(username, '')"
+    assert Searchable.create_tsvector_string('title', 'body') == \
+           "coalesce(title, '') || ' ' || coalesce(body, '')"
+    assert Searchable.create_tsvector_string('alpha', 'beta', 'gamma') == \
+           "coalesce(alpha, '') || ' ' || coalesce(beta, '') || ' ' || " \
+           "coalesce(gamma, '')"

tests/shared/fixtures.py

@@ -36,7 +36,7 @@
 from threading import Thread
 from uuid import uuid4
 from webdriver_manager.chrome import ChromeDriverManager
-from webdriver_manager.core.os_manager import ChromeType
+# from webdriver_manager.core.os_manager import ChromeType
 
 
 redis_path = which('redis-server')