#935 Added option for tagging/excluding filters

Initial changes - still need to add code to handle multiple fields in fq.

Author: nickdos

README.md

@@ -5,7 +5,7 @@ Master branch [![Build Status](https://travis-ci.com/AtlasOfLivingAustralia/bioc
 
 Occurrence & mapping webservices.
 
-Theses services are documented here https://api.ala.org.au/apps/biocache
+These services are documented here https://api.ala.org.au/apps/biocache
 
 ## Versions
 

src/main/java/au/org/ala/biocache/dao/SearchDAOImpl.java

@@ -1097,6 +1097,20 @@ private SearchResultDTO processSolrResponse(SearchRequestDTO params, QueryRespon
         SolrDocumentList sdl = qr.getResults();
         // Iterator it = qr.getResults().iterator() // Use for download
         List<FacetField> facets = qr.getFacetFields();
+        NamedList<List<PivotField>> facetPivot = qr.getFacetPivot();
+
+        // Add the (fake) facet pivot fields to the facets, as a workaround for tagging/excluding bug in solrj
+        // @see au.org.ala.biocache.util.QueryFormatUtils.applyFilterTagging comment for more details
+        if (params.getIncludeUnfilteredFacetValues() && facetPivot != null) {
+            for (Map.Entry<String, List<PivotField>> entry : facetPivot) {
+                FacetField pivotFacet = new FacetField(entry.getKey());
+                for (PivotField pivot : entry.getValue()) {
+                    pivotFacet.add(pivot.getValue().toString(), pivot.getCount());
+                }
+                facets.add(pivotFacet);
+            }
+        }
+
         List<FacetField> facetDates = qr.getFacetDates();
         Map<String, Integer> facetQueries = qr.getFacetQuery();
         if (facetDates != null) {
@@ -1365,6 +1379,10 @@ public SolrQuery initSolrQuery(SpatialSearchRequestDTO searchParams, boolean sub
                 }
             }
 
+            for (String facet : searchParams.getFacetPivots()) {
+                solrQuery.addFacetPivotField(facet);
+            }
+
             solrQuery.setFacetMinCount(1);
             solrQuery.setFacetLimit(searchParams.getFlimit());
             //include this so that the default fsort is still obeyed.

src/main/java/au/org/ala/biocache/dto/SearchRequestDTO.java

@@ -55,6 +55,7 @@ public class SearchRequestDTO {
      * Initialised with the default facets to use
      */
     protected String[] facets =  new String[0]; //FacetThemes.getAllFacetsLimited();
+    protected String[] facetPivots =  new String[0];
     protected Integer facetsMax = 30; //FacetThemes.getFacetsMax();
     /** To disable facets */
     protected Boolean facet = true; //FacetThemes.getFacetDefault();
@@ -78,6 +79,11 @@ public class SearchRequestDTO {
     private String displayString;
 
     protected Boolean includeMultivalues = false;
+    /** Flag to activate filter/tagging: facet results will include values and counts for
+     * unfiltered fields - see SOLR "tagging and excluding filters".
+     * See https://solr.apache.org/guide/8_4/faceting.html#tagging-and-excluding-filters
+     */
+    protected Boolean includeUnfilteredFacetValues = false;
 
     /**  The query context to be used for the search.  This will be used to generate extra query filters based on the search technology */
     protected String qc = "";
@@ -389,6 +395,28 @@ public void setFacets(String[] facets) {
         this.facets = list.toArray(new String[0]);
     }
 
+    public String[] getFacetPivots() {
+        return facetPivots;
+    }
+
+    public void setFacetPivots(String[] facets) {
+        QueryFormatUtils.assertNoSensitiveValues(SearchRequestDTO.class, "facets", facets);
+
+        if (facets != null && facets.length == 1 && facets[0].contains(",")) facets = facets[0].split(",");
+
+        //remove empty facets
+        List<String> list = new ArrayList<String>();
+        if (facets != null) {
+            for (String f : facets) {
+                //limit facets terms
+                if (StringUtils.isNotEmpty(f) && list.size() < facetsMax) {
+                    list.add(f);
+                }
+            }
+        }
+        this.facetPivots = list.toArray(new String[0]);
+    }
+
     public Integer getFlimit() {
         return flimit;
     }
@@ -482,6 +510,14 @@ public void setIncludeMultivalues(Boolean includeMultivalues) {
         this.includeMultivalues = includeMultivalues;
     }
 
+    public Boolean getIncludeUnfilteredFacetValues() {
+        return includeUnfilteredFacetValues;
+    }
+
+    public void setIncludeUnfilteredFacetValues(Boolean includeUnfilteredFacetValues) {
+        this.includeUnfilteredFacetValues = includeUnfilteredFacetValues;
+    }
+
     public String[] getFormattedFq() {
         return formattedFq;
     }

src/main/java/au/org/ala/biocache/dto/SpatialSearchRequestParams.java

@@ -85,6 +85,9 @@ public class SpatialSearchRequestParams {
     @Parameter(name="includeMultivalues", description = "Include multi values")
     protected Boolean includeMultivalues = false;
 
+//    @Parameter(name="includeUnfilteredFacetValues", description = "Include facet values for all available options, when filtering on the same field")
+    protected Boolean includeUnfilteredFacetValues = false;
+
     @Parameter(name="qc", description = "The query context to be used for the search. " +
             "This will be used to generate extra query filters.")
     protected String qc = "";

src/main/java/au/org/ala/biocache/util/QueryFormatUtils.java

@@ -139,6 +139,11 @@ public Map[] formatSearchQuery(SpatialSearchRequestDTO searchParams, boolean for
             //reset formattedFq in case of searchParams reuse
             searchParams.setFormattedFq(null);
 
+            // Apply filter tagging and excluding filters if flag is set
+            if (searchParams.getIncludeUnfilteredFacetValues()) {
+                applyFilterTagging(searchParams);
+            }
+
             //format fqs for facets that need ranges substituted
             if (searchParams.getFq() != null) {
                 for (int i = 0; i < searchParams.getFq().length; i++) {
@@ -184,16 +189,62 @@ public Map[] formatSearchQuery(SpatialSearchRequestDTO searchParams, boolean for
 
             //add spatial query term for wkt or lat/lon/radius parameters. DisplayString is already added by formatGeneral
             String spatialQuery = buildSpatialQueryString(searchParams);
+
             if (StringUtils.isNotEmpty(spatialQuery)) {
                 addFormattedFq(new String[] { spatialQuery }, searchParams);
             }
+
             updateQualityProfileContext(searchParams);
         }
 
         updateQueryContext(searchParams);
         return fqMaps;
     }
 
+    /**
+     * Apply facet tagging and filter exclusions to the search request.
+     *
+     * Note: due to bug/feature in SOLRJ, the filtered facets are added to
+     * the facet pivot list instead of the facet list, otherwise SOLRJ will
+     * ignore the facets with counts greater than totalRecords count, when generating
+     * the facetResults.
+     *
+     * @author "Nick dos Remedios <[email protected]>"
+     * @date 2025-01-09
+     *
+     * @param searchParams
+     */
+    private void applyFilterTagging(SpatialSearchRequestDTO searchParams) {
+        List<String> facetList = new ArrayList<String>();
+        List<String> facetPivotList = new ArrayList<String>();
+        List<String> fqList = new ArrayList<String>();
+
+        for (String f : searchParams.getFacets()) {
+            if (searchParams.getFq() != null && Arrays.stream(searchParams.getFq()).anyMatch(fq -> fq.contains(f) && !fq.contains("*"))) {
+                String prefix = "{!ex=" + f + "}";
+                facetPivotList.add(prefix + f);
+            } else {
+                facetList.add(f);
+            }
+        }
+
+        if (searchParams.getFq() != null) {
+            for (String fq : searchParams.getFq()) {
+                String fqField = org.apache.commons.lang3.StringUtils.substringBefore(fq, ":");
+                if (Arrays.asList(searchParams.getFacets()).contains(fqField) && !fq.contains("*")) {
+                    String prefix = "{!tag=" + fqField + "}";
+                    fqList.add(prefix + fq);
+                } else {
+                    fqList.add(fq);
+                }
+            }
+        }
+
+        searchParams.setFacetPivots(facetPivotList.toArray(new String[0]));
+        searchParams.setFacets(facetList.toArray(new String[0]));
+        searchParams.setFq(fqList.toArray(new String[0]));
+    }
+
     /**
      * To retrieve the key from a fq
      *
@@ -248,6 +299,7 @@ public void addFqs(String [] fqs, SpatialSearchRequestDTO searchParams) {
             }
         }
     }
+
     private void addFormattedFq(String [] fqs, SearchRequestDTO searchParams) {
         if (fqs != null && searchParams != null) {
             String[] currentFqs = searchParams.getFormattedFq();