Skip to content

Commit

Permalink
Merge pull request #1069 from AtlasOfLivingAustralia/feature/issue1067
Browse files Browse the repository at this point in the history
Feature/issue1067
  • Loading branch information
chrisala authored Feb 14, 2025
2 parents 6080b0c + 32f042c commit e7c09fd
Show file tree
Hide file tree
Showing 10 changed files with 317 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ class OrganisationService implements DataBinder {
def org = Organisation.findByOrganisationId(id)
if (org) {
List toAggregate = Score.findAllByScoreIdInList(scoreIds)
List outputSummary = reportService.organisationSummary(id, toAggregate, approvedOnly, aggregationConfig) ?: []
List outputSummary = reportService.organisationSummary(org, toAggregate, approvedOnly, aggregationConfig) ?: []

return outputSummary
} else {
Expand Down
11 changes: 7 additions & 4 deletions grails-app/services/au/org/ala/ecodata/ReportService.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -529,17 +529,20 @@ class ReportService {

/**
* Returns aggregated scores for a specified project.
* @param organisationId the organisation of interest.
* @param organisation the organisation of interest.
* @param aggregationSpec defines the scores to be aggregated and if any grouping needs to occur.
* [{score:{name: , units:, aggregationType}, groupBy: {entity: <one of 'activity', 'output', 'organisation', 'site>, property: String <the entity property to group by>}, ...]
*
* @return the results of the aggregation. The results will be a List of Maps, the structure of each Map is
* described in @see au.org.ala.ecodata.reporting.Aggregation.results()
*
*/
List organisationSummary(String organisationId, List aggregationSpec, boolean approvedActivitiesOnly = false, Map topLevelAggregationConfig = null) {
List organisationSummary(Organisation organisation, List aggregationSpec, boolean approvedActivitiesOnly = false, Map topLevelAggregationConfig = null) {

List activities = activityService.findAllForOrganisationId(organisationId, 'FLAT')
aggregate(activities, aggregationSpec, approvedActivitiesOnly, topLevelAggregationConfig)
List activities = activityService.findAllForOrganisationId(organisation.organisationId, 'FLAT')
// Make the organisation available to the aggregation as one of the Scores
// needs access to it.
List aggregationData = activities.collect{it+[organisation:organisation]}
aggregate(aggregationData, aggregationSpec, approvedActivitiesOnly, topLevelAggregationConfig)
}
}
65 changes: 65 additions & 0 deletions src/main/groovy/au/org/ala/ecodata/metadata/ExpressionUtil.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package au.org.ala.ecodata.metadata

import groovy.util.logging.Slf4j
import org.springframework.context.expression.MapAccessor
import org.springframework.expression.AccessException
import org.springframework.expression.EvaluationContext
import org.springframework.expression.Expression
import org.springframework.expression.TypedValue
import org.springframework.expression.spel.support.StandardEvaluationContext
import org.springframework.lang.Nullable
import org.springframework.util.Assert

@Slf4j
class ExpressionUtil {


/** Evaluates the supplied expression against the data in the supplied Map */
static def evaluateWithDefault(Expression expression, Map expressionContext, defaultValue) {
StandardEvaluationContext context = new StandardEvaluationContext(expressionContext)
context.addPropertyAccessor(new NoExceptionMapAccessor(null))

def result
try {
result = expression.getValue(context)
if (Double.isNaN(result)) {
result = defaultValue
}
}
catch (Exception e) {
log.debug("Error evaluating expression: ${expression.getExpressionString()}", e.getMessage())
result = defaultValue
}
result

}


/**
* Extends the Spring MapAccessor but instead of throwing an Exception if the
* Map does not have a property with the supplied name just returns null.
*/
static class NoExceptionMapAccessor extends MapAccessor {

TypedValue defaultValue
NoExceptionMapAccessor(defaultValue) {
this.defaultValue = new TypedValue(defaultValue)
}
@Override
public TypedValue read(EvaluationContext context, @Nullable Object target, String name) throws AccessException {
Assert.state(target instanceof Map, "Target must be of type Map");
Map<?, ?> map = (Map<?, ?>) target
Object value = map.get(name)
if (value == null && !map.containsKey(name)) {
return defaultValue
}
return new TypedValue(value)

}

@Override
public boolean canRead(EvaluationContext context, @Nullable Object target, String name) throws AccessException {
return target instanceof Map
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,14 @@ class FilteredAggregationConfig extends AggregationConfig {
}
}

class ExpressionAggregationConfig extends CompositeAggregationConfig {
/** The expression to evaluate */
String expression

/** The value to return if the expression evaluation fails (e.g. missing variables due to no data, divide by 0) */
def defaultValue
}

class GroupingConfig extends Aggregation {
String type // DATE, DISCRETE, FILTER, HISTOGRAM
Object filterValue
Expand All @@ -36,6 +44,10 @@ class Aggregation extends AggregationConfig {
String property
}

class DistinctAggregationConfig extends Aggregation {
String keyProperty
}

class AggregationResult {
String label
int count
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
package au.org.ala.ecodata.reporting

/**
* An AggregationBuilder can create instances of the Aggregator class based on the supplied information about
* how the aggregation should be performed.
Expand All @@ -10,8 +9,10 @@ class AggregatorFactory {
if (config instanceof GroupingAggregationConfig) {
new GroupingAggregator(config)
}
else if (config instanceof ExpressionAggregationConfig) {
new ExpressionAggregator(config)
}
else if (config instanceof CompositeAggregationConfig) {

new CompositeAggregator(config)
}
else if (config instanceof FilteredAggregationConfig) {
Expand All @@ -36,6 +37,11 @@ class AggregatorFactory {

configObject = new GroupingAggregationConfig(label:config.label, groups: groupingConfig, childAggregations: createChildConfig(config))
}
else if (config.containsKey('expression')) {
configObject = new ExpressionAggregationConfig(
expression:config.expression, label:config.label, childAggregations: createChildConfig(config)
)
}
else if (config.containsKey('filter')) {
GroupingConfig filterConfig = new GroupingConfig(config.filter)

Expand All @@ -44,6 +50,9 @@ class AggregatorFactory {
else if (config.containsKey('childAggregations')) {
configObject = new CompositeAggregationConfig(label:config.label, childAggregations: createChildConfig(config))
}
else if (config.keyProperty) {
configObject = new DistinctAggregationConfig(config)
}
else {
configObject = new Aggregation(config)
}
Expand Down Expand Up @@ -77,11 +86,14 @@ class AggregatorFactory {
return new Aggregators.AverageAggregator(config.label, config.property)
break;
case Score.AGGREGATION_TYPE.HISTOGRAM.name():
return new Aggregators.HistogramAggregator(config.label, config.property)
return new Aggregators. HistogramAggregator(config.label, config.property)
break;
case Score.AGGREGATION_TYPE.SET.name():
return new Aggregators.SetAggregator(config.label, config.property)
break;
case Score.AGGREGATION_TYPE.DISTINCT_SUM.name():
return new Aggregators.DistinctSumAggregator(config.label, config.property, ((DistinctAggregationConfig)config).keyProperty)
break
default:
throw new IllegalAccessException("Invalid aggregation type: ${config.type}, label:${config.label}, property:${config.property}")
}
Expand Down
29 changes: 29 additions & 0 deletions src/main/groovy/au/org/ala/ecodata/reporting/Aggregators.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -184,4 +184,33 @@ class Aggregators {
}
}

/**
* The DistinctSumAggregator will sum the values of a property where a second property is distinct.
* The use case for this is to sum data in values that appear more than once during aggregation.
* For example, most aggregation is done with Output data, and each one contains the Activity and owner (e.g. Project)
* data as well.
* The specific use case for this is to sum the data from Organisation reports and divide by the total
* funding, which is a property on the Organisation object.
* The keyProperty is the property that is used to determine if the value is distinct. The case of the
* organisation funding amount, the organisationId would be used as the key property.
*/
static class DistinctSumAggregator extends SummingAggegrator {
protected PropertyAccessor keyPropertyAccessor
protected Set seenKeys
DistinctSumAggregator(String label, String property, String keyProperty) {
super(label, property)
keyPropertyAccessor = new PropertyAccessor(keyProperty)
seenKeys = new HashSet()
}

void aggregateSingle(Map data) {
def key = keyPropertyAccessor.getPropertyValue(data)
if (!seenKeys.contains(key)) {
seenKeys.add(key)
super.aggregateSingle(data)
}
}

}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package au.org.ala.ecodata.reporting

import au.org.ala.ecodata.metadata.ExpressionUtil
import org.springframework.expression.Expression
import org.springframework.expression.ExpressionParser
import org.springframework.expression.spel.standard.SpelExpressionParser

/**
* Categorises an activity into a group based on a supplied grouping criteria then delegates to the appropriate
* Aggregator.
*/
class ExpressionAggregator extends BaseAggregator {

List<AggregatorIf> aggregators
int count

AggregatorFactory factory = new AggregatorFactory()
Expression expression
def defaultValue = 0

ExpressionAggregator(ExpressionAggregationConfig config) {

ExpressionParser expressionParser = new SpelExpressionParser()
expression = expressionParser.parseExpression(config.expression)
defaultValue = config.defaultValue ?: 0

aggregators = config.childAggregations.collect {
factory.createAggregator(it)
}
}

PropertyAccessor getPropertyAccessor() {
return null
}

void aggregateSingle(Map output) {
count++
aggregators.each {
it.aggregate(output)
}

}

/**
* If we have a single childAggregation, return a SingleResult, otherwise a
* GroupedAggregationResult.
*/
AggregationResult result() {

SingleResult result = new SingleResult()
Map expressionContext = [:]

aggregators.each {
AggregationResult childResult = it.result()
if (childResult instanceof SingleResult) {
expressionContext[childResult.label] = childResult.result
}
else if (childResult instanceof GroupedAggregationResult) {
expressionContext[childResult.label] = childResult.groups?.collectEntries {
[(it.group):it.results[0]?.result]
}
}
}

result.result = ExpressionUtil.evaluateWithDefault(expression, expressionContext, defaultValue)

result
}

}


2 changes: 1 addition & 1 deletion src/main/groovy/au/org/ala/ecodata/reporting/Score.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ package au.org.ala.ecodata.reporting
class Score {

/** Enumerates the currently supported ways to aggregate output scores. */
enum AGGREGATION_TYPE {SUM, AVERAGE, COUNT, HISTOGRAM, SET}
enum AGGREGATION_TYPE {SUM, AVERAGE, COUNT, HISTOGRAM, SET, DISTINCT_SUM}

/** The name of the output to which the score belongs */
String outputName
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package au.org.ala.ecodata.reporting

import spock.lang.Specification

class DistinctSumAggregatorSpec extends Specification {

// write a test for the DistinctSumAggregator
def "The DistinctSumAggregator can sum the distinct values of a property"() {
given:
Map config = [
label:"value1",
"property": "data.value1",
"type": "DISTINCT_SUM",
"keyProperty": "data.group"
]
Aggregators.DistinctSumAggregator aggregator = new AggregatorFactory().createAggregator(config)

when:
aggregator.aggregate([data:[value1:1, group: "group1"]])
aggregator.aggregate([data:[value1:1, group: "group1"]])
aggregator.aggregate([data:[value1:2, group: "group1"]])
aggregator.aggregate([data:[value1:2, group: "group2"]])
aggregator.aggregate([data:[value1:3, group: "group3"]])

AggregationResult result = aggregator.result()

then:
result.result == 6
}
}
Loading

0 comments on commit e7c09fd

Please sign in to comment.