Feature/ogc 508 replace elastic search by postgres text search

src/onegov/activity/models/attendee.py

@@ -1,4 +1,5 @@
 from datetime import date
+
 from onegov.activity.models.booking import Booking
 from onegov.core.orm import Base
 from onegov.core.orm.mixins import TimestampMixin
@@ -11,7 +12,6 @@
 from sqlalchemy import Date
 from sqlalchemy import Float
 from sqlalchemy import ForeignKey
-from sqlalchemy import Index
 from sqlalchemy import Integer
 from sqlalchemy import Numeric
 from sqlalchemy import Text
@@ -43,6 +43,10 @@ class Attendee(Base, TimestampMixin, ORMSearchable):
     }
     es_public = False
 
+    @property
+    def search_score(self):
+        return 3
+
     @property
     def es_suggestion(self):
         return self.name
@@ -179,7 +183,3 @@ def happiness(cls, period_id):
         order_by='Booking.created',
         backref='attendee'
     )
-
-    __table_args__ = (
-        Index('unique_child_name', 'username', 'name', unique=True),
-    )

src/onegov/agency/models/person.py

@@ -21,8 +21,6 @@ def es_public(self):
         'title': {'type': 'text'},
         'function': {'type': 'localized'},
         'email': {'type': 'text'},
-        'phone_internal': {'type': 'text'},
-        'phone_es': {'type': 'text'}
     }
 
     @property

src/onegov/agency/views/search.py

@@ -1,13 +1,23 @@
 from onegov.agency import AgencyApp
 from onegov.agency.layout import AgencySearchLayout
 from onegov.core.security import Public
-from onegov.org.models import Search
+from onegov.org.models import Search, SearchPostgres
 from onegov.org.views.search import search as search_view
+from onegov.org.views.search import search_postgres as search_postgres_view
 
 
 @AgencyApp.html(model=Search, template='search.pt', permission=Public)
-def search(self, request):
+def agency_search(self, request):
     data = search_view(self, request)
     if isinstance(data, dict):
         data['layout'] = AgencySearchLayout(self, request)
     return data
+
+
+@AgencyApp.html(model=SearchPostgres, template='search_postgres.pt',
+                permission=Public)
+def agency_search_postgres(self, request):
+    data = search_postgres_view(self, request)
+    if isinstance(data, dict):
+        data['layout'] = AgencySearchLayout(self, request)
+    return data

src/onegov/core/upgrade.py

@@ -480,6 +480,12 @@ def has_column(self, table: str, column: str) -> bool:
             table, schema=self.schema
         )}
 
+    def has_index(self, table: str, index: str) -> bool:
+        inspector = Inspector(self.operations_connection)
+        return index in {i['name'] for i in inspector.get_indexes(
+            table, schema=self.schema
+        )}
+
     def has_enum(self, enum: str) -> bool:
         return self.session.execute(f"""
             SELECT EXISTS (

src/onegov/directory/models/directory.py

@@ -17,10 +17,8 @@
 from onegov.file.utils import as_fileintent
 from onegov.form import flatten_fieldsets, parse_formcode, parse_form
 from onegov.search import SearchableContent
-from sqlalchemy import Column
+from sqlalchemy import Column, Text, Integer
 from sqlalchemy import func, exists, and_
-from sqlalchemy import Integer
-from sqlalchemy import Text
 from sqlalchemy.orm import object_session
 from sqlalchemy.orm import relationship
 from sqlalchemy.orm.attributes import InstrumentedAttribute
@@ -103,6 +101,10 @@ def count(self):
         backref='directory'
     )
 
+    @property
+    def search_score(self):
+        return 7
+
     @property
     def entry_cls_name(self):
         return 'DirectoryEntry'

src/onegov/directory/models/directory_entry.py

@@ -6,12 +6,13 @@
 from onegov.file import AssociatedFiles
 from onegov.gis import CoordinatesMixin
 from onegov.search import SearchableContent
-from sqlalchemy import Column
+from sqlalchemy import Column, cast
 from sqlalchemy import ForeignKey
 from sqlalchemy import Index
 from sqlalchemy import Text
 from sqlalchemy.dialects.postgresql import HSTORE
 from sqlalchemy.ext.mutable import MutableDict
+from sqlalchemy.ext.hybrid import hybrid_property
 from uuid import uuid4
 
 
@@ -25,7 +26,7 @@ class DirectoryEntry(Base, ContentMixin, CoordinatesMixin, TimestampMixin,
         'keywords': {'type': 'keyword'},
         'title': {'type': 'localized'},
         'lead': {'type': 'localized'},
-        'directory_id': {'type': 'keyword'},
+        '_directory_id': {'type': 'keyword'},
 
         # since the searchable text might include html, we remove it
         # even if there's no html -> possibly decreasing the search
@@ -86,6 +87,10 @@ def external_link_title(self):
     def external_link_visible(self):
         return self.directory.configuration.link_visible
 
+    @hybrid_property
+    def _directory_id(self):
+        return cast(self.directory_id, Text)
+
     @property
     def directory_name(self):
         return self.directory.name
@@ -98,7 +103,7 @@ def keywords(self):
     def keywords(self, value):
         self._keywords = {k: '' for k in value} if value else None
 
-    @property
+    @hybrid_property
     def text(self):
         return self.directory.configuration.extract_searchable(self.values)
 

src/onegov/event/models/event.py

@@ -6,6 +6,7 @@
 from icalendar import Calendar as vCalendar
 from icalendar import Event as vEvent
 from icalendar import vRecur
+from sqlalchemy.ext.hybrid import hybrid_property
 
 from onegov.core.orm import Base
 from onegov.core.orm.abstract import associated
@@ -109,6 +110,10 @@ class Event(Base, OccurrenceMixin, TimestampMixin, SearchableContent,
         EventFile, 'pdf', 'one-to-one', uselist=False, backref_suffix='pdf'
     )
 
+    @property
+    def search_score(self):
+        return 1
+
     def set_image(self, content, filename=None):
         self.set_blob('image', content, filename)
 
@@ -148,9 +153,16 @@ def set_blob(self, blob, content, filename=None):
         'description': {'type': 'localized'},
         'location': {'type': 'localized'},
         'organizer': {'type': 'localized'},
-        'filter_keywords': {'type': 'keyword'}
     }
 
+    @hybrid_property
+    def description(self):  # noqa: F811
+        return self.content['description'].astext
+
+    @hybrid_property
+    def organizer(self):  # noqa: F811
+        return self.content['organizer'].astext
+
     @property
     def es_public(self):
         return self.state == 'published'

src/onegov/feriennet/models/activity.py

@@ -1,4 +1,7 @@
 from functools import cached_property
+
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.activity import Activity, ActivityCollection, Occasion
 from onegov.activity import PublicationRequestCollection
 from onegov.activity.models import DAYS
@@ -24,6 +27,10 @@ class VacationActivity(Activity, CoordinatesExtension, SearchableContent):
         'organiser': {'type': 'text'}
     }
 
+    @property
+    def search_score(self):
+        return 1
+
     @property
     def es_public(self):
         return self.state == 'accepted'
@@ -32,7 +39,7 @@ def es_public(self):
     def es_skip(self):
         return self.state == 'preview'
 
-    @property
+    @hybrid_property
     def organiser(self):
         organiser = [
             self.user.username,

src/onegov/feriennet/templates/mail_booking_accepted.pt

@@ -16,7 +16,7 @@
             <tal:b tal:content="structure bookings_link"></tal:b>
         </p>
         <br>
-        <p i18n:translate>Best regards</p>                    
+        <p i18n:translate>Best regards</p>
         <metal:b use-macro="layout.macros['sender']|nothing" />
     </tal:b>
 </div>

src/onegov/feriennet/templates/mail_booking_canceled.pt

@@ -15,7 +15,7 @@
             <tal:b tal:content="structure bookings_link"></tal:b>
         </p>
         <br>
-        <p i18n:translate>Best regards</p>                    
+        <p i18n:translate>Best regards</p>
         <metal:b use-macro="layout.macros['sender']|nothing" />
     </tal:b>
 </div>

src/onegov/file/models/file.py

@@ -6,6 +6,7 @@
 from contextlib import contextmanager
 from collections import defaultdict
 from depot.fields.sqlalchemy import UploadedFileField as UploadedFileFieldBase
+
 from onegov.core.crypto import random_token
 from onegov.core.orm import Base
 from onegov.core.orm.abstract import Associable
@@ -262,6 +263,10 @@ class File(Base, Associable, TimestampMixin):
         Index('files_by_type_and_order', 'type', 'order'),
     )
 
+    @property
+    def search_score(self) -> int:
+        return 10
+
     @hybrid_property
     def signature_timestamp(self) -> 'datetime | None':
         if self.signed:

src/onegov/file/models/fileset.py

@@ -6,6 +6,7 @@
 
 
 from typing import TYPE_CHECKING
+
 if TYPE_CHECKING:
     from .file import File
 

src/onegov/form/models/definition.py

@@ -140,6 +140,9 @@ class FormDefinition(Base, ContentMixin, TimestampMixin, Extendable):
     }
 
     @property
+    def search_score(self) -> int:
+        return 7
+
     def form_class(self) -> Type['Form']:
         """ Parses the form definition and returns a form class. """
 

src/onegov/fsi/models/course.py

@@ -1,4 +1,5 @@
 from arrow import utcnow
+
 from onegov.core.html import html_to_text
 from onegov.core.orm import Base
 from onegov.core.orm.types import UUID
@@ -32,6 +33,10 @@ class Course(Base, ORMSearchable):
     # hides the course in the collection for non-admins
     hidden_from_public = Column(Boolean, nullable=False, default=False)
 
+    @property
+    def search_score(self):
+        return 2
+
     @property
     def title(self):
         return self.name

src/onegov/fsi/models/course_attendee.py

@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.core.orm import Base
 from onegov.core.orm.types import UUID, JSON
 from sqlalchemy import Boolean
@@ -107,7 +109,7 @@ def __str__(self):
         cascade='all, delete-orphan'
     )
 
-    @property
+    @hybrid_property
     def title(self):
         return ' '.join((
             p for p in (
@@ -131,7 +133,7 @@ def role(self):
             return 'member'
         return self.user.role
 
-    @property
+    @hybrid_property
     def email(self):
         """Needs a switch for external users"""
         if not self.user_id:

src/onegov/fsi/models/course_event.py

@@ -22,13 +22,15 @@
 from onegov.fsi.models.course_subscription import subscription_table
 from onegov.search import ORMSearchable
 
-
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from .course import Course
     from .course_notification_template import (
-        CancellationTemplate, CourseNotificationTemplate, InfoTemplate,
-        ReminderTemplate, SubscriptionTemplate
+        CancellationTemplate,
+        CourseNotificationTemplate,
+        InfoTemplate,
+        ReminderTemplate,
+        SubscriptionTemplate
     )
 
 COURSE_EVENT_STATUSES = ('created', 'confirmed', 'canceled', 'planned')
@@ -83,7 +85,7 @@ def es_public(self):
     def title(self):
         return str(self)
 
-    @property
+    @hybrid_property
     def name(self):
         return self.course.name
 
@@ -95,7 +97,7 @@ def lead(self):
             f'{self.presenter_company}'
         )
 
-    @property
+    @hybrid_property
     def description(self):
         return self.course.description
 

src/onegov/fsi/views/search.py

@@ -1,15 +1,29 @@
 from onegov.core.security import Personal
 from onegov.fsi import FsiApp
-from onegov.org.models import Search
+from onegov.org.models import Search, SearchPostgres
 from onegov.org.views.search import search as search_view
+from onegov.org.views.search import search_postgres as search_postgres_view
 from onegov.org.views.search import suggestions as suggestions_view
+from onegov.org.views.search import suggestions_postgres as \
+    suggestions_postgres_view
 
 
 @FsiApp.html(model=Search, template='search.pt', permission=Personal)
 def search(self, request):
     return search_view(self, request)
 
 
+@FsiApp.html(model=SearchPostgres, template='search_postgres.pt',
+             permission=Personal)
+def search_postgres(self, request):
+    return search_postgres_view(self, request)
+
+
 @FsiApp.json(model=Search, name='suggest', permission=Personal)
 def suggestions(self, request):
     return suggestions_view(self, request)
+
+
+@FsiApp.json(model=SearchPostgres, name='suggest', permission=Personal)
+def suggestions_postgres(self, request):
+    return suggestions_postgres_view(self, request)