diff --git a/README.md b/README.md index 1011fca5a..22c151cae 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Master branch [![Build Status](https://travis-ci.com/AtlasOfLivingAustralia/bioc Occurrence & mapping webservices. -Theses services are documented here https://api.ala.org.au/apps/biocache +These services are documented here https://api.ala.org.au/apps/biocache ## Versions diff --git a/build.gradle b/build.gradle index 8e01ec8eb..ff67edaec 100644 --- a/build.gradle +++ b/build.gradle @@ -40,7 +40,7 @@ repositories { } group = 'au.org.ala' -version = '3.5.1-SNAPSHOT' +version = '3.6.0-SNAPSHOT' boolean inplace = false diff --git a/src/main/java/au/org/ala/biocache/dao/SearchDAOImpl.java b/src/main/java/au/org/ala/biocache/dao/SearchDAOImpl.java index 5f4e9fe1c..ab858f004 100644 --- a/src/main/java/au/org/ala/biocache/dao/SearchDAOImpl.java +++ b/src/main/java/au/org/ala/biocache/dao/SearchDAOImpl.java @@ -1097,6 +1097,22 @@ private SearchResultDTO processSolrResponse(SearchRequestDTO params, QueryRespon SolrDocumentList sdl = qr.getResults(); // Iterator it = qr.getResults().iterator() // Use for download List facets = qr.getFacetFields(); + NamedList> facetPivot = qr.getFacetPivot(); + + // Add the (fake) facet pivot fields to the facets, as a workaround for tagging/excluding bug in solrj + // @see au.org.ala.biocache.util.QueryFormatUtils.applyFilterTagging comment for more details + if (params.getIncludeUnfilteredFacetValues() && facetPivot != null) { + for (Map.Entry> entry : facetPivot) { + FacetField pivotFacet = new FacetField(entry.getKey()); + for (PivotField pivot : entry.getValue()) { + if (pivot.getValue() != null) { + pivotFacet.add(pivot.getValue().toString(), pivot.getCount()); + } + } + facets.add(pivotFacet); + } + } + List facetDates = qr.getFacetDates(); Map facetQueries = qr.getFacetQuery(); if (facetDates != null) { @@ -1365,6 +1381,10 @@ public SolrQuery initSolrQuery(SpatialSearchRequestDTO searchParams, boolean sub } } + for (String facet : searchParams.getPivotFacets()) { + solrQuery.addFacetPivotField(facet); + } + solrQuery.setFacetMinCount(1); solrQuery.setFacetLimit(searchParams.getFlimit()); //include this so that the default fsort is still obeyed. diff --git a/src/main/java/au/org/ala/biocache/dto/SearchRequestDTO.java b/src/main/java/au/org/ala/biocache/dto/SearchRequestDTO.java index 575e3d097..1bf94c2fe 100644 --- a/src/main/java/au/org/ala/biocache/dto/SearchRequestDTO.java +++ b/src/main/java/au/org/ala/biocache/dto/SearchRequestDTO.java @@ -55,6 +55,7 @@ public class SearchRequestDTO { * Initialised with the default facets to use */ protected String[] facets = new String[0]; //FacetThemes.getAllFacetsLimited(); + protected String[] pivotFacets = new String[0]; protected Integer facetsMax = 30; //FacetThemes.getFacetsMax(); /** To disable facets */ protected Boolean facet = true; //FacetThemes.getFacetDefault(); @@ -78,6 +79,11 @@ public class SearchRequestDTO { private String displayString; protected Boolean includeMultivalues = false; + /** Flag to activate filter/tagging: facet results will include values and counts for + * unfiltered fields - see SOLR "tagging and excluding filters". + * See https://solr.apache.org/guide/8_4/faceting.html#tagging-and-excluding-filters + */ + protected Boolean includeUnfilteredFacetValues = false; /** The query context to be used for the search. This will be used to generate extra query filters based on the search technology */ protected String qc = ""; @@ -389,6 +395,28 @@ public void setFacets(String[] facets) { this.facets = list.toArray(new String[0]); } + public String[] getPivotFacets() { + return pivotFacets; + } + + public void setPivotFacets(String[] facets) { + QueryFormatUtils.assertNoSensitiveValues(SearchRequestDTO.class, "facets", facets); + + if (facets != null && facets.length == 1 && facets[0].contains(",")) facets = facets[0].split(","); + + //remove empty facets + List list = new ArrayList(); + if (facets != null) { + for (String f : facets) { + //limit facets terms + if (StringUtils.isNotEmpty(f) && list.size() < facetsMax) { + list.add(f); + } + } + } + this.pivotFacets = list.toArray(new String[0]); + } + public Integer getFlimit() { return flimit; } @@ -482,6 +510,14 @@ public void setIncludeMultivalues(Boolean includeMultivalues) { this.includeMultivalues = includeMultivalues; } + public Boolean getIncludeUnfilteredFacetValues() { + return includeUnfilteredFacetValues; + } + + public void setIncludeUnfilteredFacetValues(Boolean includeUnfilteredFacetValues) { + this.includeUnfilteredFacetValues = includeUnfilteredFacetValues; + } + public String[] getFormattedFq() { return formattedFq; } diff --git a/src/main/java/au/org/ala/biocache/dto/SpatialSearchRequestParams.java b/src/main/java/au/org/ala/biocache/dto/SpatialSearchRequestParams.java index 58ceed026..d8c350801 100644 --- a/src/main/java/au/org/ala/biocache/dto/SpatialSearchRequestParams.java +++ b/src/main/java/au/org/ala/biocache/dto/SpatialSearchRequestParams.java @@ -85,6 +85,9 @@ public class SpatialSearchRequestParams { @Parameter(name="includeMultivalues", description = "Include multi values") protected Boolean includeMultivalues = false; +// @Parameter(name="includeUnfilteredFacetValues", description = "Include facet values for all available options, when filtering on the same field") + protected Boolean includeUnfilteredFacetValues = false; + @Parameter(name="qc", description = "The query context to be used for the search. " + "This will be used to generate extra query filters.") protected String qc = ""; diff --git a/src/main/java/au/org/ala/biocache/util/QueryFormatUtils.java b/src/main/java/au/org/ala/biocache/util/QueryFormatUtils.java index dc7da196c..85e35dbfa 100644 --- a/src/main/java/au/org/ala/biocache/util/QueryFormatUtils.java +++ b/src/main/java/au/org/ala/biocache/util/QueryFormatUtils.java @@ -23,6 +23,7 @@ import java.util.regex.MatchResult; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import static au.org.ala.biocache.dto.OccurrenceIndex.CONTAINS_SENSITIVE_PATTERN; import static java.util.stream.Collectors.joining; @@ -89,6 +90,14 @@ public class QueryFormatUtils { @Value("${solr.circle.segments:18}") int solrCircleSegments = 18; + /** + * When facet tagging/excluding is enabled, this flag determines whether to combine + * multiple excluded facet fields into each facet {!ex} tag. + * @see au.org.ala.biocache.util.QueryFormatUtils#applyFilterTagging(au.org.ala.biocache.dto.SpatialSearchRequestDTO) + */ + @Value("${facets.combineExcludedFields:false}") + boolean combineExcludedFacetFields = false; + /** * This is appended to the query displayString when SpatialSearchRequestParams.wkt is used. */ @@ -130,7 +139,7 @@ public Map[] formatSearchQuery(SpatialSearchRequestDTO searchParams, boolean for //Only format the query if it doesn't already supply a formattedQuery. if (forceQueryFormat || StringUtils.isEmpty(searchParams.getFormattedQuery())) { - String [] originalFqs = searchParams.getFq(); + String[] originalFqs = Arrays.copyOf(searchParams.getFq(), searchParams.getFq().length); // copy by value String [] formatted = formatQueryTerm(searchParams.getQ(), searchParams); searchParams.setDisplayString(formatted[0]); @@ -139,13 +148,20 @@ public Map[] formatSearchQuery(SpatialSearchRequestDTO searchParams, boolean for //reset formattedFq in case of searchParams reuse searchParams.setFormattedFq(null); + // Apply filter tagging and excluding filters if flag is set + if (searchParams.getIncludeUnfilteredFacetValues()) { + applyFilterTagging(searchParams); + } + //format fqs for facets that need ranges substituted if (searchParams.getFq() != null) { for (int i = 0; i < searchParams.getFq().length; i++) { String fq = searchParams.getFq()[i]; + String fqOriginal = originalFqs[i]; // not altered by `applyFilterTagging()` - if (fq != null && fq.length() > 0) { + if (fq != null && !fq.isEmpty()) { formatted = formatQueryTerm(fq, searchParams); + String[] formattedOriginal = formatQueryTerm(fqOriginal, searchParams); if (StringUtils.isNotEmpty(formatted[1])) { addFormattedFq(new String[]{formatted[1]}, searchParams); @@ -155,21 +171,21 @@ public Map[] formatSearchQuery(SpatialSearchRequestDTO searchParams, boolean for //do not add spatial fields if (originalFqs != null && i < originalFqs.length && !formatted[1].contains(spatialField + ":")) { Facet facet = new Facet(); - facet.setDisplayName(formatted[0]); - String[] fv = fq.split(":"); + facet.setDisplayName(formattedOriginal[0]); + String[] fv = fqOriginal.split(":"); if (fv.length >= 2) { facet.setName(fv[0]); - facet.setValue(fq.substring(fv[0].length() + 1)); + facet.setValue(fqOriginal.substring(fv[0].length() + 1)); } activeFacetMap.put(facet.getName(), facet); // activeFacetMap is based on the assumption that each fq is on different filter so its a [StringKey: Facet] structure // but actually different fqs can use same filter key for example &fq=-month:'11'&fq=-month='12' so we added a new map // activeFacetObj which is [StringKey: List] - String fqKey = parseFQ(fq); + String fqKey = parseFQ(fqOriginal); if (fqKey != null) { - Facet fct = new Facet(fqKey, formatted[0]); // display name is the formatted name, for example '11' to 'November' - fct.setValue(fq); // value in activeFacetMap is the part with key replaced by '', but here is the original fq because front end will need it + Facet fct = new Facet(fqKey, formattedOriginal[0]); // display name is the formatted name, for example '11' to 'November' + fct.setValue(fqOriginal); // value in activeFacetMap is the part with key replaced by '', but here is the original fq because front end will need it List valList = activeFacetObj.getOrDefault(fqKey, new ArrayList<>()); valList.add(fct); activeFacetObj.put(fqKey, valList); @@ -184,9 +200,11 @@ public Map[] formatSearchQuery(SpatialSearchRequestDTO searchParams, boolean for //add spatial query term for wkt or lat/lon/radius parameters. DisplayString is already added by formatGeneral String spatialQuery = buildSpatialQueryString(searchParams); + if (StringUtils.isNotEmpty(spatialQuery)) { addFormattedFq(new String[] { spatialQuery }, searchParams); } + updateQualityProfileContext(searchParams); } @@ -194,6 +212,58 @@ public Map[] formatSearchQuery(SpatialSearchRequestDTO searchParams, boolean for return fqMaps; } + /** + * Apply facet tagging and filter exclusions to the search request. + * + * Note: due to bug/feature in SOLRJ, the excluded facets are added to + * the facet pivot list instead of the facet list, otherwise SOLRJ will + * ignore the facets with counts greater than totalRecords count, when generating + * the facetResults. + * + * @author "Nick dos Remedios " + * @date 2025-01-09 + * + * @param searchParams + */ + private void applyFilterTagging(SpatialSearchRequestDTO searchParams) { + List facetList = new ArrayList(); + List facetPivotList = new ArrayList(); + List fqList = new ArrayList(); + + // Get a list of excluded fields + List excludedFields = Arrays.stream(searchParams.getFacets()) + .filter(f -> searchParams.getFq() != null && Arrays.stream(searchParams.getFq()).anyMatch(fq -> fq.contains(f))) + .collect(Collectors.toList()); + + // Add the excluded fields to the facetPivotList || facetList + for (String f : searchParams.getFacets()) { + if (excludedFields.contains(f)) { + String prefix = combineExcludedFacetFields ? "{!ex=" + String.join(",", excludedFields) + "}" : "{!ex=" + f + "}"; + facetPivotList.add(prefix + f); + } else { + facetList.add(f); + } + } + + // Add the tag syntax to the fqList, if the fq is a facet + if (searchParams.getFq() != null) { + for (String fq : searchParams.getFq()) { + String fqField = org.apache.commons.lang3.StringUtils.substringBefore(fq, ":"); + if (Arrays.asList(searchParams.getFacets()).contains(fqField)) { + String prefix = "{!tag=" + fqField + "}"; + fqList.add(prefix + fq); + } else { + fqList.add(fq); + } + } + } + + // Update the searchParams with the new facetPivotList, facetList, and fqList + searchParams.setPivotFacets(facetPivotList.toArray(new String[0])); + searchParams.setFacets(facetList.toArray(new String[0])); + searchParams.setFq(fqList.toArray(new String[0])); + } + /** * To retrieve the key from a fq * @@ -248,6 +318,7 @@ public void addFqs(String [] fqs, SpatialSearchRequestDTO searchParams) { } } } + private void addFormattedFq(String [] fqs, SearchRequestDTO searchParams) { if (fqs != null && searchParams != null) { String[] currentFqs = searchParams.getFormattedFq(); diff --git a/src/test/java/au/org/ala/biocache/util/QueryFormatUtilsSpec.groovy b/src/test/java/au/org/ala/biocache/util/QueryFormatUtilsSpec.groovy index 5763aaabd..50968d741 100644 --- a/src/test/java/au/org/ala/biocache/util/QueryFormatUtilsSpec.groovy +++ b/src/test/java/au/org/ala/biocache/util/QueryFormatUtilsSpec.groovy @@ -1,6 +1,13 @@ package au.org.ala.biocache.util +import au.org.ala.biocache.dao.QidCacheDAO +import au.org.ala.biocache.dto.Qid +import au.org.ala.biocache.dto.SpatialSearchRequestDTO +import au.org.ala.biocache.service.AuthService +import au.org.ala.biocache.service.DataQualityService +import au.org.ala.biocache.service.LayersService import au.org.ala.biocache.service.ListsService +import au.org.ala.biocache.util.solr.FieldMappingUtil import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.ObjectMapper import com.google.common.io.Resources @@ -14,11 +21,22 @@ class QueryFormatUtilsSpec extends Specification { def listsService = Stub(ListsService) def searchUtils = Stub(SearchUtils) + def layersService = Stub(LayersService) + def qidCacheDao = Stub(QidCacheDAO) + def dataQualityService = Stub(DataQualityService) + def authService = Stub(AuthService) + def fieldMappingUtil = Stub(FieldMappingUtil) def setup() { - queryFormatUtils.listsService = listsService queryFormatUtils.searchUtils = searchUtils + queryFormatUtils.layersService = layersService + queryFormatUtils.qidCacheDao = qidCacheDao + queryFormatUtils.dataQualityService = dataQualityService + queryFormatUtils.fieldMappingUtil = Mock(FieldMappingUtil) { + translateQueryFields(_) >> { String query -> return query } + } + queryFormatUtils.authService = authService } @@ -69,6 +87,103 @@ class QueryFormatUtilsSpec extends Specification { 'before species_list:dr123 between species_list:dr456 after' | 'field:before species_list:dr123 field:between species_list:dr456 field:after' || 'before dr123 (FAILED) between dr456 (FAILED) after' | 'field:before (NOT *:*) field:between (NOT *:*) field:after' } + def "test formatSearchQuery with empty query"() { + given: + SpatialSearchRequestDTO searchParams = new SpatialSearchRequestDTO() + searchParams.setQ("") + searchParams.setFq(new String[0]) + + when: + def result = queryFormatUtils.formatSearchQuery(searchParams, false) + + then: + result[0].isEmpty() + result[1].isEmpty() + searchParams.getFormattedQuery() == "" + searchParams.getDisplayString() == "" + } + + def "test formatSearchQuery with simple query"() { + given: + SpatialSearchRequestDTO searchParams = new SpatialSearchRequestDTO() + searchParams.setQ("taxon_name:Test") + searchParams.setFq(new String[0]) + + when: + def result = queryFormatUtils.formatSearchQuery(searchParams, false) + + then: + result[0].isEmpty() + result[1].isEmpty() + searchParams.getFormattedQuery() == "taxon_name:Test" + searchParams.getDisplayString() == "taxon_name:Test" + } + + def "test formatSearchQuery with qid"() { + given: + SpatialSearchRequestDTO searchParams = new SpatialSearchRequestDTO() + searchParams.setQ("qid:123") + searchParams.setFq(new String[0]) + qidCacheDao.get(_) >> new Qid(q: "taxon_name:Test", fqs: new String[0]) + + when: + def result = queryFormatUtils.formatSearchQuery(searchParams, false) + + then: + result[0].isEmpty() + result[1].isEmpty() + searchParams.getFormattedQuery() == "taxon_name:Test" + searchParams.getDisplayString() == "taxon_name:Test" + } + + def "test formatSearchQuery with facets"() { + given: + SpatialSearchRequestDTO searchParams = new SpatialSearchRequestDTO() + searchParams.setQ("taxon_name:Test") + searchParams.setFacet(true) + searchParams.setIncludeUnfilteredFacetValues(false) + searchParams.setFq(new String[]{"month:1", "year:2020"}) + searchParams.setFacets(new String[]{"month", "year", "eventDate"}) + + when: + def result = queryFormatUtils.formatSearchQuery(searchParams, false) + + then: + result[0].size() == 2 + result[1].size() == 2 + searchParams.getFormattedQuery() == "taxon_name:Test" + searchParams.getFormattedFq() == new String[]{"month:1", "year:2020"} + searchParams.getFq() == new String[]{"month:1", "year:2020"} + searchParams.getDisplayString() == "taxon_name:Test" + searchParams.getFacets() == new String[]{"month", "year", "eventDate"} + searchParams.getPivotFacets() == new String[]{} + } + + + + def "test formatSearchQuery with tagging and excluded facets"() { + given: + SpatialSearchRequestDTO searchParams = new SpatialSearchRequestDTO() + searchParams.setQ("taxon_name:Test") + searchParams.setFacet(true) + searchParams.setIncludeUnfilteredFacetValues(true) + searchParams.setFq(new String[]{"month:1", "year:2020"}) + searchParams.setFacets(new String[]{"month", "year", "eventDate"}) + + when: + def result = queryFormatUtils.formatSearchQuery(searchParams, false) + + then: + result[0].size() == 2 + result[1].size() == 2 + searchParams.getFormattedQuery() == "taxon_name:Test" + searchParams.getDisplayString() == "taxon_name:Test" + searchParams.getFormattedFq() == new String[]{"{!tag=month}month:1", "{!tag=year}year:2020"} + searchParams.getFq() == new String[]{"month:1", "year:2020"} + searchParams.getFacets() == new String[]{"eventDate"} + searchParams.getPivotFacets() == new String[]{"{!ex=month,year}month", "{!ex=month,year}year"} + } + private static ObjectMapper om = new ObjectMapper() private static String getResultQuery(String uid) {