Skip to content

Commit

Permalink
Adding IUCN RedList category to the species match cache (#19)
Browse files Browse the repository at this point in the history 
* Adding the IUCN RedList Category to the name match cache
gbif/pipelines#257

* Checking possible null values in the NameUsageMatch response and in WS responses
  • Loading branch information
@fmendezh
fmendezh authored Feb 11, 2021
1 parent 17e95b4 commit e8ae095
Show file tree
Hide file tree
Showing 6 changed files with 110 additions and 43 deletions.
28 changes: 16 additions & 12 deletions kvs-gbif/src/main/java/org/gbif/kvs/species/NameUsageMatchKVStoreFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
import org.gbif.kvs.hbase.Command;
import org.gbif.kvs.hbase.HBaseStore;
import org.gbif.rest.client.configuration.ClientConfiguration;
import org.gbif.rest.client.species.NameMatchService;
import org.gbif.rest.client.species.ChecklistbankService;
import org.gbif.rest.client.species.NameUsageMatch;
import org.gbif.rest.client.species.retrofit.NameMatchServiceSyncClient;
import org.gbif.rest.client.species.retrofit.ChecklistbankServiceSyncClient;

import java.io.IOException;
import java.util.Objects;
Expand Down Expand Up @@ -100,16 +100,19 @@ public static BiFunction<byte[], NameUsageMatch, Put> valueMutator(byte[] column

public static KeyValueStore<SpeciesMatchRequest, NameUsageMatch> nameUsageMatchKVStore(CachedHBaseKVStoreConfiguration configuration,
ClientConfiguration clientConfiguration) throws IOException {
NameMatchServiceSyncClient nameMatchServiceSyncClient = new NameMatchServiceSyncClient(clientConfiguration);
ChecklistbankServiceSyncClient
checklistbankServiceSyncClient = new ChecklistbankServiceSyncClient(clientConfiguration);
Command closeHandler = () -> {
try {
nameMatchServiceSyncClient.close();
checklistbankServiceSyncClient.close();
} catch (IOException ex) {
throw logAndThrow(ex, "Error closing client");
}
};
KeyValueStore<SpeciesMatchRequest, NameUsageMatch> keyValueStore = Objects.nonNull(configuration.getHBaseKVStoreConfiguration())?
hbaseKVStore(configuration, nameMatchServiceSyncClient, closeHandler) : restKVStore(nameMatchServiceSyncClient, closeHandler);
hbaseKVStore(configuration,
checklistbankServiceSyncClient, closeHandler) : restKVStore(
checklistbankServiceSyncClient, closeHandler);
if (Objects.nonNull(configuration.getCacheCapacity())) {
return KeyValueCache.cache(keyValueStore, configuration.getCacheCapacity(), SpeciesMatchRequest.class, NameUsageMatch.class);
}
Expand All @@ -127,7 +130,7 @@ public static KeyValueStore<SpeciesMatchRequest, NameUsageMatch> nameUsageMatchK


private static KeyValueStore<SpeciesMatchRequest, NameUsageMatch> hbaseKVStore(CachedHBaseKVStoreConfiguration configuration,
NameMatchService nameMatchService,
ChecklistbankService checklistbankService,
Command closeHandler) throws IOException {
return HBaseStore.<SpeciesMatchRequest, NameUsageMatch, NameUsageMatch>builder()
.withHBaseStoreConfiguration(configuration.getHBaseKVStoreConfiguration())
Expand All @@ -146,7 +149,7 @@ private static KeyValueStore<SpeciesMatchRequest, NameUsageMatch> hbaseKVStore(C
.withLoader(
request -> {
try {
return nameMatchService.match(
return checklistbankService.match(
request.getKingdom(),
request.getPhylum(),
request.getClazz(),
Expand All @@ -166,10 +169,11 @@ private static KeyValueStore<SpeciesMatchRequest, NameUsageMatch> hbaseKVStore(C
}

private static KeyValueStore<SpeciesMatchRequest, NameUsageMatch> restKVStore(ClientConfiguration clientConfiguration) {
NameMatchServiceSyncClient nameMatchServiceSyncClient = new NameMatchServiceSyncClient(clientConfiguration);
return restKVStore(nameMatchServiceSyncClient, () -> {
ChecklistbankServiceSyncClient
checklistbankServiceSyncClient = new ChecklistbankServiceSyncClient(clientConfiguration);
return restKVStore(checklistbankServiceSyncClient, () -> {
try {
nameMatchServiceSyncClient.close();
checklistbankServiceSyncClient.close();
} catch (IOException ex) {
throw logAndThrow(ex, "Error closing client");
}
Expand All @@ -179,13 +183,13 @@ private static KeyValueStore<SpeciesMatchRequest, NameUsageMatch> restKVStore(Cl
/**
* Builds a KV Store backed by the rest client.
*/
private static KeyValueStore<SpeciesMatchRequest, NameUsageMatch> restKVStore(NameMatchService nameMatchService, Command closeHandler) {
private static KeyValueStore<SpeciesMatchRequest, NameUsageMatch> restKVStore(ChecklistbankService checklistbankService, Command closeHandler) {
return new KeyValueStore<SpeciesMatchRequest, NameUsageMatch>() {

@Override
public NameUsageMatch get(SpeciesMatchRequest key) {
try {
return nameMatchService.match(
return checklistbankService.match(
key.getKingdom(),
key.getPhylum(),
key.getClazz(),
Expand Down
61 changes: 47 additions & 14 deletions kvs-indexing/src/main/java/org/gbif/kvs/indexing/species/NameUsageMatchIndexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,22 @@

import org.apache.beam.sdk.coders.AvroCoder;
import org.apache.beam.sdk.io.AvroIO;

import org.gbif.api.util.VocabularyUtils;
import org.gbif.api.vocabulary.Rank;
import org.gbif.api.vocabulary.ThreatStatus;
import org.gbif.kvs.SaltedKeyGenerator;
import org.gbif.kvs.conf.CachedHBaseKVStoreConfiguration;
import org.gbif.kvs.indexing.options.ConfigurationMapper;
import org.gbif.kvs.species.NameUsageMatchKVStoreFactory;
import org.gbif.kvs.species.SpeciesMatchRequest;
import org.gbif.kvs.species.TaxonParsers;
import org.gbif.rest.client.configuration.ClientConfiguration;
import org.gbif.rest.client.species.NameMatchService;
import org.gbif.rest.client.species.ChecklistbankService;
import org.gbif.rest.client.species.NameUsageMatch;
import org.gbif.rest.client.species.retrofit.NameMatchServiceSyncClient;
import org.gbif.rest.client.species.retrofit.ChecklistbankServiceSyncClient;

import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.function.BiFunction;
Expand Down Expand Up @@ -105,34 +109,63 @@ private static void run(NameUsageMatchIndexingOptions options) {
new SaltedKeyGenerator(
storeConfiguration.getHBaseKVStoreConfiguration().getNumOfKeyBuckets());

private transient NameMatchService nameMatchService;
private transient ChecklistbankService checklistbankService;

private transient BiFunction<byte[], NameUsageMatch, Put> valueMutator;

@Setup
public void start() {
nameMatchService = new NameMatchServiceSyncClient(nameMatchClientConfiguration);
checklistbankService = new ChecklistbankServiceSyncClient(nameMatchClientConfiguration);
valueMutator =
NameUsageMatchKVStoreFactory.valueMutator(
Bytes.toBytes(storeConfiguration.getHBaseKVStoreConfiguration().getColumnFamily()),
Bytes.toBytes(storeConfiguration.getValueColumnQualifier()));
}

/**
* Gets the first non-null value between the accepted usage and the usage of a NameUsageMatch.
*/
private Integer getAcceptedUsageOrUsage(NameUsageMatch nameUsageMatch) {
if (Objects.nonNull(nameUsageMatch.getAcceptedUsage())) {
return nameUsageMatch.getAcceptedUsage().getKey();
} else if (Objects.nonNull(nameUsageMatch.getUsage())) {
return nameUsageMatch.getUsage().getKey();
}
return null;
}

/**
* Retrieves the IUCN Red List category from the name usage match response.
*/
private ThreatStatus getIucnRedListCategory(NameUsageMatch nameUsageMatch) {
Integer usageKey = getAcceptedUsageOrUsage(nameUsageMatch);
if (Objects.nonNull(usageKey)) {
Map<String, String> response = checklistbankService.getIucnRedListCategory(usageKey);
if (Objects.nonNull(response)) {
Optional<ThreatStatus> threatStatus =
VocabularyUtils.lookup(response.get("category"), ThreatStatus.class);
return threatStatus.orElse(null);
}
}
return null;
}

@ProcessElement
public void processElement(ProcessContext context) {
try {
SpeciesMatchRequest request = context.element();
NameUsageMatch nameUsageMatch = nameMatchService.match(request.getKingdom(),
request.getPhylum(),
request.getClazz(),
request.getOrder(),
request.getFamily(),
request.getGenus(),
Optional.ofNullable(TaxonParsers.interpretRank(request)).map(Rank::name).orElse(null),
TaxonParsers.interpretScientificName(request),
false,
false);
NameUsageMatch nameUsageMatch = checklistbankService.match(request.getKingdom(),
request.getPhylum(),
request.getClazz(),
request.getOrder(),
request.getFamily(),
request.getGenus(),
Optional.ofNullable(TaxonParsers.interpretRank(request)).map(Rank::name).orElse(null),
TaxonParsers.interpretScientificName(request),
false,
false);
if (Objects.nonNull(nameUsageMatch)) {
Optional.ofNullable(getIucnRedListCategory(nameUsageMatch)).ifPresent(nameUsageMatch::setIucnRedListCategory);
byte[] saltedKey = keyGenerator.computeKey(request.getLogicalKey());
context.output(valueMutator.apply(saltedKey, nameUsageMatch));
}
Expand Down
12 changes: 10 additions & 2 deletions ...rest/client/species/NameMatchService.java → .../client/species/ChecklistbankService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@


import java.io.Closeable;
import java.util.Map;

/**
* GBIF Backbone name match service.
* GBIF Backbone name match and IUCN RedList services.
*/
public interface NameMatchService extends Closeable {
public interface ChecklistbankService extends Closeable {

/**
* Fuzzy matches scientific names against the GBIF Backbone Taxonomy with the optional classification provided.
Expand All @@ -27,4 +28,11 @@ public interface NameMatchService extends Closeable {
NameUsageMatch match(String kingdom,String phylum, String clazz, String order, String family, String genus,
String rank, String name, boolean verbose, boolean strict);

/**
* Gets the IUCN RedList Category of a nubKey.
* @param nubKey GBIF backbone key.
* @return a possible null map containing the IUCN RedList Category
*/
Map<String,String> getIucnRedListCategory(Integer nubKey);

}
4 changes: 4 additions & 0 deletions kvs-rest-clients/src/main/java/org/gbif/rest/client/species/NameUsageMatch.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import org.gbif.api.v2.RankedName;
import org.gbif.api.vocabulary.TaxonomicStatus;
import org.gbif.api.vocabulary.ThreatStatus;

import lombok.Data;

Expand All @@ -19,6 +20,9 @@ public class NameUsageMatch implements Serializable {
private List<RankedName> classification = new ArrayList<>();
private NameUsageMatch.Diagnostics diagnostics = new NameUsageMatch.Diagnostics();

//This is not part of the NameUsageMatch response, but it is stored in the same record in the Cache
private ThreatStatus iucnRedListCategory;

@Data
public static class Diagnostics {
private org.gbif.api.model.checklistbank.NameUsageMatch.MatchType matchType;
Expand Down
16 changes: 13 additions & 3 deletions ...es/retrofit/NameMatchRetrofitService.java → ...etrofit/ChecklistbankRetrofitService.java
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,32 @@
package org.gbif.rest.client.species.retrofit;

import org.gbif.rest.client.species.ChecklistbankService;
import org.gbif.rest.client.species.NameUsageMatch;

import java.util.Map;

import retrofit2.Call;
import retrofit2.http.GET;
import retrofit2.http.Path;
import retrofit2.http.Query;

/**
* see {@link org.gbif.rest.client.species.NameMatchService}
* see {@link ChecklistbankService}
*/
public interface NameMatchRetrofitService {
public interface ChecklistbankRetrofitService {

/**
* See {@link org.gbif.rest.client.species.NameMatchService#match(String, String, String, String, String, String, String, String, boolean, boolean)}
* See {@link ChecklistbankService#match(String, String, String, String, String, String, String, String, boolean, boolean)}
*/
@GET("/v1/species/match2")
Call<NameUsageMatch> match(@Query("kingdom") String kingdom, @Query("phylum") String phylum,
@Query("class") String clazz, @Query("order") String order, @Query("family") String family,
@Query("genus") String genus, @Query("rank") String rank, @Query("name") String name,
@Query("verbose") boolean verbose, @Query("strict") boolean strict);

/**
* See {@link org.gbif.rest.client.species.ChecklistbankService#getIucnRedListCategory(Integer)}
*/
@GET("/v1/species/{nubKey}/iucnRedListCategory")
Call<Map<String,String>> getIucnRedListCategory(@Path("nubKey") Integer nubKey);
}
32 changes: 20 additions & 12 deletions .../retrofit/NameMatchServiceSyncClient.java → ...rofit/ChecklistbankServiceSyncClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import okhttp3.OkHttpClient;
import org.gbif.rest.client.configuration.ClientConfiguration;
import org.gbif.rest.client.retrofit.RetrofitClientFactory;
import org.gbif.rest.client.species.NameMatchService;
import org.gbif.rest.client.species.ChecklistbankService;
import org.gbif.rest.client.species.NameUsageMatch;


Expand All @@ -12,6 +12,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Comparator;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Stream;

Expand All @@ -20,33 +21,40 @@
/**
* Synchronous Retrofit service client for the NameMatch GBIF service.
*/
public class NameMatchServiceSyncClient implements NameMatchService {
public class ChecklistbankServiceSyncClient implements ChecklistbankService {

//Wrapped service
private final NameMatchRetrofitService nameMatchRetrofitService;
//Wrapped services
private final ChecklistbankRetrofitService checklistbankRetrofitService;

private final OkHttpClient okHttpClient;


/**
* Creates an instance using the provided configuration settings.
* @param clientConfiguration Rest client configuration
*/
public NameMatchServiceSyncClient(ClientConfiguration clientConfiguration) {
public ChecklistbankServiceSyncClient(ClientConfiguration clientConfiguration) {
okHttpClient = RetrofitClientFactory.createClient(clientConfiguration);
nameMatchRetrofitService = RetrofitClientFactory.createRetrofitClient(okHttpClient,
clientConfiguration.getBaseApiUrl(),
NameMatchRetrofitService.class);
checklistbankRetrofitService = RetrofitClientFactory.createRetrofitClient(okHttpClient,
clientConfiguration.getBaseApiUrl(),
ChecklistbankRetrofitService.class);
}

/**
* See {@link NameMatchService#match(String, String, String, String, String, String, String, String, boolean, boolean)}
* See {@link ChecklistbankService#match(String, String, String, String, String, String, String, String, boolean, boolean)}
*/
@Override
public NameUsageMatch match(String kingdom, String phylum, String clazz, String order, String family, String genus,
String rank, String name, boolean verbose, boolean strict) {
return syncCall(nameMatchRetrofitService.match(kingdom, phylum, clazz, order, family, genus, rank, name, verbose,
strict));
return syncCall(checklistbankRetrofitService.match(kingdom, phylum, clazz, order, family, genus, rank, name, verbose,
strict));
}

/**
* See {@link ChecklistbankService#getIucnRedListCategory(Integer)}
*/
@Override
public Map<String, String> getIucnRedListCategory(Integer nubKey) {
return syncCall(checklistbankRetrofitService.getIucnRedListCategory(nubKey));
}

@Override
Expand Down

0 comments on commit e8ae095

Please sign in to comment.