improved order of suggest results

gbif · Jun 17, 2024 · a4b0037 · a4b0037
1 parent c4d8d98
commit a4b0037
Show file tree

Hide file tree

Showing 6 changed files with 82 additions and 18 deletions.
diff --git a/core/src/main/resources/liquibase/013-suggest-normalize-function.xml b/core/src/main/resources/liquibase/013-suggest-normalize-function.xml
@@ -0,0 +1,19 @@
+<databaseChangeLog
+    xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-2.0.xsd">
+
+  <changeSet id="13" author="mlopez">
+    <sql splitStatements="false" stripComments="false">
+      <![CDATA[
+        CREATE OR REPLACE FUNCTION normalize_suggest(query text)
+        RETURNS text AS
+        $func$
+          BEGIN
+            RETURN lower(unaccent(trim(query)));
+          END
+        $func$ LANGUAGE plpgsql;
+      ]]>
+    </sql>
+  </changeSet>
+</databaseChangeLog>
diff --git a/core/src/main/resources/liquibase/master.xml b/core/src/main/resources/liquibase/master.xml
@@ -16,5 +16,6 @@
   <include file="liquibase/010-labels_definitions_indexes.xml" />
   <include file="liquibase/011-optional-release-url.xml" />
   <include file="liquibase/012-updated-constraint-names-format.xml" />
+  <include file="liquibase/013-suggest-normalize-function.xml" />
 
 </databaseChangeLog>
diff --git a/core/src/main/resources/org/gbif/vocabulary/persistence/mappers/ConceptMapper.xml b/core/src/main/resources/org/gbif/vocabulary/persistence/mappers/ConceptMapper.xml
@@ -158,69 +158,72 @@
     FROM (
     (
       <include refid="SUGGEST_LANG_Q">
-        <property name="q" value="cl.value = unaccent(trim(#{query}))"/>
+        <property name="q" value="normalize_suggest(cl.value) = normalize_suggest(#{query})"/>
         <property name="score" value="100"/>
       </include>
     )
     <if test="lang != null and fallbackLang != null">
       UNION ALL
       (
         <include refid="SUGGEST_FALLBACK_Q">
-          <property name="q" value="cl.value = unaccent(trim(#{query}))"/>
+          <property name="q" value="normalize_suggest(cl.value) = normalize_suggest(#{query})"/>
           <property name="score" value="100"/>
         </include>
       )
     </if>
     UNION ALL
     (
       <include refid="SUGGEST_NAME_Q">
-        <property name="q" value="c.name = unaccent(trim(#{query}))"/>
+        <property name="q" value="normalize_suggest(c.name) = normalize_suggest(#{query})"/>
         <property name="score" value="100"/>
       </include>
     )
     UNION ALL
     (
       <include refid="SUGGEST_LANG_Q">
-        <property name="q" value="cl.value ilike unaccent(trim(#{query}))||'%'"/>
+        <property name="q" value="normalize_suggest(cl.value) ilike normalize_suggest(#{query})||'%'"/>
         <property name="score" value="70"/>
       </include>
     )
     <if test="lang != null and fallbackLang != null">
       UNION ALL
       (
         <include refid="SUGGEST_FALLBACK_Q">
-          <property name="q" value="cl.value ilike unaccent(trim(#{query}))||'%'"/>
+          <property name="q" value="normalize_suggest(cl.value) ilike  normalize_suggest(#{query})||'%'"/>
           <property name="score" value="70"/>
         </include>
       )
     </if>
     UNION ALL
     (
       <include refid="SUGGEST_NAME_Q">
-        <property name="q" value="c.name ilike unaccent(trim(#{query}))||'%'"/>
+        <property name="q" value="normalize_suggest(c.name) ilike normalize_suggest(#{query})||'%'"/>
         <property name="score" value="70"/>
       </include>
     )
     UNION ALL
     (
       <include refid="SUGGEST_LANG_Q">
-        <property name="q" value="cl.value ilike '%'||unaccent(trim(#{query}))||'%' AND NOT cl.value ilike unaccent(trim(#{query}))||'%'"/>
+        <property name="q" value="normalize_suggest(cl.value) ilike '%'||normalize_suggest(#{query})||'%'
+                                AND NOT normalize_suggest(cl.value) ilike normalize_suggest(#{query})||'%'"/>
         <property name="score" value="50"/>
       </include>
     )
     <if test="lang != null and fallbackLang != null">
       UNION ALL
       (
         <include refid="SUGGEST_FALLBACK_Q">
-          <property name="q" value="cl.value ilike '%'||unaccent(trim(#{query}))||'%' AND NOT cl.value ilike unaccent(trim(#{query}))||'%'"/>
+          <property name="q" value="normalize_suggest(cl.value) ilike '%'||normalize_suggest(#{query})||'%'
+                          AND NOT normalize_suggest(cl.value) ilike normalize_suggest(#{query})||'%'"/>
           <property name="score" value="50"/>
         </include>
       )
     </if>
     UNION ALL
     (
       <include refid="SUGGEST_NAME_Q">
-        <property name="q" value="c.name ilike '%'||unaccent(trim(#{query}))||'%' AND NOT c.name ilike unaccent(trim(#{query}))||'%'"/>
+        <property name="q" value="normalize_suggest(c.name) ilike '%'||normalize_suggest(#{query})||'%'
+                          AND NOT normalize_suggest(c.name) ilike normalize_suggest(#{query})||'%'"/>
         <property name="score" value="50"/>
       </include>
      )

diff --git a/core/src/main/resources/org/gbif/vocabulary/persistence/mappers/VocabularyMapper.xml b/core/src/main/resources/org/gbif/vocabulary/persistence/mappers/VocabularyMapper.xml
@@ -197,7 +197,7 @@
     FROM (
       (
         <include refid="SUGGEST_LANG_Q">
-          <property name="q" value="vl.value = unaccent(trim(#{query}))"/>
+          <property name="q" value="normalize_suggest(vl.value) = normalize_suggest(#{query})"/>
           <property name="score" value="100"/>
         </include>
       )
@@ -213,53 +213,56 @@
       UNION ALL
       (
         <include refid="SUGGEST_NAME_Q">
-          <property name="q" value="v.name = unaccent(trim(#{query}))"/>
+          <property name="q" value="normalize_suggest(v.name) = normalize_suggest(#{query})"/>
           <property name="score" value="100"/>
         </include>
       )
       UNION ALL
       (
         <include refid="SUGGEST_LANG_Q">
-          <property name="q" value="vl.value ilike unaccent(trim(#{query}))||'%'"/>
+          <property name="q" value="normalize_suggest(vl.value) ilike normalize_suggest(#{query})||'%'"/>
           <property name="score" value="70"/>
         </include>
       )
       <if test="lang != null and fallbackLang != null">
         UNION ALL
         (
           <include refid="SUGGEST_FALLBACK_Q">
-            <property name="q" value="vl.value ilike unaccent(trim(#{query}))||'%'"/>
+            <property name="q" value="normalize_suggest(vl.value) ilike normalize_suggest(#{query})||'%'"/>
             <property name="score" value="70"/>
           </include>
         )
       </if>
       UNION ALL
       (
         <include refid="SUGGEST_NAME_Q">
-          <property name="q" value="v.name ilike unaccent(trim(#{query}))||'%'"/>
+          <property name="q" value="normalize_suggest(v.name) ilike normalize_suggest(#{query})||'%'"/>
           <property name="score" value="70"/>
         </include>
         )
       UNION ALL
       (
         <include refid="SUGGEST_LANG_Q">
-          <property name="q" value="vl.value ilike '%'||unaccent(trim(#{query}))||'%' AND NOT vl.value ilike unaccent(trim(#{query}))||'%'"/>
+          <property name="q" value="normalize_suggest(vl.value) ilike '%'||normalize_suggest(#{query})||'%'
+                                    AND NOT normalize_suggest(vl.value) ilike normalize_suggest(#{query})||'%'"/>
           <property name="score" value="50"/>
         </include>
       )
       <if test="lang != null and fallbackLang != null">
         UNION ALL
         (
         <include refid="SUGGEST_FALLBACK_Q">
-          <property name="q" value="vl.value ilike '%'||unaccent(trim(#{query}))||'%' AND NOT vl.value ilike unaccent(trim(#{query}))||'%'"/>
+          <property name="q" value="normalize_suggest(vl.value) ilike '%'||normalize_suggest(#{query})||'%'
+                                    AND NOT normalize_suggest(vl.value) ilike normalize_suggest(#{query})||'%'"/>
           <property name="score" value="50"/>
         </include>
         )
       </if>
       UNION ALL
       (
         <include refid="SUGGEST_NAME_Q">
-          <property name="q" value="v.name ilike '%'||unaccent(trim(#{query}))||'%' AND NOT v.name ilike unaccent(trim(#{query}))||'%'"/>
+          <property name="q" value="normalize_suggest(v.name) ilike '%'||normalize_suggest(#{query})||'%'
+                          AND NOT normalize_suggest(v.name) ilike normalize_suggest(#{query})||'%'"/>
           <property name="score" value="50"/>
         </include>
       )

diff --git a/core/src/test/java/org/gbif/vocabulary/persistence/mappers/ConceptMapperTest.java b/core/src/test/java/org/gbif/vocabulary/persistence/mappers/ConceptMapperTest.java
@@ -225,6 +225,14 @@ public void suggestTest() {
         c2.getKey(),
         Label.builder().language(LanguageRegion.ENGLISH).value("Label").createdBy("test").build());
 
+    conceptMapper.addLabel(
+            c2.getKey(),
+            Label.builder()
+                    .language(LanguageRegion.PORTUGUESE)
+                    .value("tílDE")
+                    .createdBy("test")
+                    .build());
+
     // check result values
     List<SuggestDto> result =
         conceptMapper.suggest("suggest1", c1.getVocabularyKey(), null, null, DEFAULT_SUGGEST_LIMIT);
@@ -258,7 +266,21 @@ public void suggestTest() {
         c2.getKey(),
         conceptMapper
             .suggest("Label", c1.getVocabularyKey(), LanguageRegion.ENGLISH, null, 1)
-            .get(0).getKey());
+            .get(0)
+            .getKey());
+
+    assertEquals(
+        c2.getKey(),
+        conceptMapper
+            .suggest("TILDE", c1.getVocabularyKey(), LanguageRegion.PORTUGUESE, null, 1)
+            .get(0)
+            .getKey());
+    assertEquals(
+        c2.getKey(),
+        conceptMapper
+            .suggest("Tílde", c1.getVocabularyKey(), LanguageRegion.PORTUGUESE, null, 1)
+            .get(0)
+            .getKey());
 
     Concept c3 = createNewEntity();
     c3.setVocabularyKey(vocabularies[1].getKey());

diff --git a/core/src/test/java/org/gbif/vocabulary/persistence/mappers/VocabularyMapperTest.java b/core/src/test/java/org/gbif/vocabulary/persistence/mappers/VocabularyMapperTest.java
@@ -224,6 +224,14 @@ public void suggestTest() {
         v2.getKey(),
         Label.builder().language(LanguageRegion.ENGLISH).value("Label").createdBy("test").build());
 
+    vocabularyMapper.addLabel(
+        v2.getKey(),
+        Label.builder()
+            .language(LanguageRegion.PORTUGUESE)
+            .value("tílDE")
+            .createdBy("test")
+            .build());
+
     // check result values
     List<SuggestDto> result =
         vocabularyMapper.suggest("suggest1", null, null, DEFAULT_SUGGEST_LIMIT);
@@ -271,6 +279,14 @@ public void suggestTest() {
     assertEquals(
         v2.getKey(),
         vocabularyMapper.suggest("Label", LanguageRegion.ENGLISH, null, 1).get(0).getKey());
+
+    assertEquals(
+        v2.getKey(),
+        vocabularyMapper.suggest("TILDE", LanguageRegion.PORTUGUESE, null, 1).get(0).getKey());
+
+    assertEquals(
+        v2.getKey(),
+        vocabularyMapper.suggest("Tílde", LanguageRegion.PORTUGUESE, null, 1).get(0).getKey());
   }
 
   @Test