Adding IUCN RedList Category to taxon record, hdfs record and Elastic…

…search record and schema #257
gbif · Mar 1, 2021 · dd0d575 · dd0d575
1 parent 289a579
commit dd0d575
Show file tree

Hide file tree

Showing 9 changed files with 35 additions and 4 deletions.
diff --git a/gbif/pipelines/ingest-gbif-beam/src/main/resources/elasticsearch/es-occurrence-schema.json b/gbif/pipelines/ingest-gbif-beam/src/main/resources/elasticsearch/es-occurrence-schema.json
@@ -321,7 +321,8 @@
               "uninomial": {"type": "text", "index":  false}
             }
           },
-          "verbatimScientificName": {"type": "keyword", "boost": 100.0, "copy_to": "all", "normalizer": "lowercase_normalizer"}
+          "verbatimScientificName": {"type": "keyword", "boost": 100.0, "copy_to": "all", "normalizer": "lowercase_normalizer"},
+          "iucnRedListCategory": {"type": "keyword"}
         }
       },
 

diff --git a/pom.xml b/pom.xml
@@ -80,7 +80,7 @@
     <gbif-api.version>0.138</gbif-api.version>
     <gbif-common.version>0.47</gbif-common.version>
     <dwc-api.version>1.28</dwc-api.version>
-    <kvs.version>1.13</kvs.version>
+    <kvs.version>1.17-SNAPSHOT</kvs.version>
     <hbase-utils.version>0.12</hbase-utils.version>
     <gbif-wrangler.version>0.3</gbif-wrangler.version>
     <gbif-occurrence.version>0.140</gbif-occurrence.version>

diff --git a/sdks/core/src/main/java/org/gbif/pipelines/core/converters/GbifJsonConverter.java b/sdks/core/src/main/java/org/gbif/pipelines/core/converters/GbifJsonConverter.java
@@ -478,6 +478,7 @@ private BiConsumer<JsonConverter, SpecificRecordBase> getTaxonomyRecordConverter
               .setUsage(trOrg.getUsage())
               .setUsageParsedName(trOrg.getUsageParsedName())
               .setDiagnostics(trOrg.getDiagnostics())
+              .setIucnRedListCategory(trOrg.getIucnRedListCategory())
               .setIssues(null); // Issues are accumulated
 
       if (!skipId) {

diff --git a/.../core/src/main/java/org/gbif/pipelines/core/converters/OccurrenceHdfsRecordConverter.java b/.../core/src/main/java/org/gbif/pipelines/core/converters/OccurrenceHdfsRecordConverter.java
@@ -292,6 +292,10 @@ private static BiConsumer<OccurrenceHdfsRecord, SpecificRecordBase> taxonMapper(
           .ifPresent(d -> hr.setTaxonomicstatus(d.name()));
 
       setCreatedIfGreater(hr, tr.getCreated());
+
+      Optional.ofNullable(tr.getIucnRedListCategory())
+          .ifPresent(u -> hr.setIucnRedListCategory(u.name()));
+
       addIssues(tr.getIssues(), hr);
     };
   }

diff --git a/sdks/core/src/main/java/org/gbif/pipelines/core/parsers/taxonomy/TaxonRecordConverter.java b/sdks/core/src/main/java/org/gbif/pipelines/core/parsers/taxonomy/TaxonRecordConverter.java
@@ -13,6 +13,7 @@
 import org.gbif.pipelines.io.avro.RankedName;
 import org.gbif.pipelines.io.avro.Status;
 import org.gbif.pipelines.io.avro.TaxonRecord;
+import org.gbif.pipelines.io.avro.ThreatStatus;
 import org.gbif.rest.client.species.NameUsageMatch;
 
 /** Adapts a {@link NameUsageMatch} into a {@link TaxonRecord} */
@@ -45,6 +46,11 @@ private static TaxonRecord convertInternal(NameUsageMatch source, TaxonRecord ta
     taxonRecord.setNomenclature(convertNomenclature(source.getNomenclature()));
     taxonRecord.setDiagnostics(convertDiagnostics(source.getDiagnostics()));
 
+    // IUCN Red List Category
+    Optional.ofNullable(source.getIucnRedListCategory())
+        .map(cat -> ThreatStatus.valueOf(cat.name()))
+        .ifPresent(taxonRecord::setIucnRedListCategory);
+
     return taxonRecord;
   }
 

diff --git a/sdks/core/src/test/java/org/gbif/pipelines/core/converters/GbifJsonConverterTest.java b/sdks/core/src/test/java/org/gbif/pipelines/core/converters/GbifJsonConverterTest.java
@@ -12,6 +12,7 @@
 import org.gbif.pipelines.io.avro.*;
 import org.gbif.pipelines.io.avro.MediaType;
 import org.gbif.pipelines.io.avro.Rank;
+import org.gbif.pipelines.io.avro.ThreatStatus;
 import org.gbif.pipelines.io.avro.grscicoll.GrscicollRecord;
 import org.gbif.pipelines.io.avro.grscicoll.Match;
 import org.junit.Assert;
@@ -506,6 +507,7 @@ public void taxonRecordUsageTest() {
             + "\"acceptedUsage\":{\"key\":2,"
             + "\"name\":\"Name2\","
             + "\"rank\":\"ABERRATION\"},"
+            + "\"iucnRedListCategory\":\"CRITICALLY_ENDANGERED\","
             + "\"chemoformKey\":1,"
             + "\"chemoform\":\"Name\","
             + "\"aberrationKey\":2,"
@@ -531,6 +533,7 @@ public void taxonRecordUsageTest() {
                 RankedName.newBuilder().setKey(1).setName("n").setRank(Rank.ABERRATION).build())
             .setClassification(rankedNameList)
             .setAcceptedUsage(name2)
+            .setIucnRedListCategory(ThreatStatus.CRITICALLY_ENDANGERED)
             .build();
 
     ExtendedRecord extendedRecord =

diff --git a/...e/src/test/java/org/gbif/pipelines/core/converters/OccurrenceHdfsRecordConverterTest.java b/...e/src/test/java/org/gbif/pipelines/core/converters/OccurrenceHdfsRecordConverterTest.java
@@ -53,6 +53,7 @@
 import org.gbif.pipelines.io.avro.State;
 import org.gbif.pipelines.io.avro.TaxonRecord;
 import org.gbif.pipelines.io.avro.TemporalRecord;
+import org.gbif.pipelines.io.avro.ThreatStatus;
 import org.gbif.pipelines.io.avro.grscicoll.GrscicollRecord;
 import org.gbif.pipelines.io.avro.grscicoll.Match;
 import org.junit.Assert;
@@ -138,6 +139,7 @@ public void extendedRecordMapperTest() {
                 2L) // This value for lastParsed and lastInterpreted since is greater that the Basic
             // record created date
             .setClassification(classification)
+            .setIucnRedListCategory(ThreatStatus.CRITICALLY_ENDANGERED)
             .build();
 
     TemporalRecord temporalRecord =
@@ -175,6 +177,7 @@ public void extendedRecordMapperTest() {
     Assert.assertEquals(OccurrenceStatus.ABSENT.name(), hdfsRecord.getVOccurrencestatus());
     Assert.assertEquals("0", hdfsRecord.getVIndividualcount());
     Assert.assertEquals("2000/2010", hdfsRecord.getVEventdate());
+    Assert.assertEquals(ThreatStatus.CRITICALLY_ENDANGERED.name(), hdfsRecord.getIucnRedListCategory());
 
     // Test fields names with reserved words
     Assert.assertEquals("CLASS", hdfsRecord.getClass$());

diff --git a/sdks/models/src/main/avro/core/taxon-record.avsc b/sdks/models/src/main/avro/core/taxon-record.avsc
@@ -50,6 +50,15 @@
       "noIssue", "noMatch", "parentChildSynonym", "questionSpecies", "speciesPlural"
      ]
   },
+  {
+     "name" : "ThreatStatus",
+     "namespace" : "org.gbif.pipelines.io.avro",
+     "type" : "enum",
+     "symbols" : [
+        "EXTINCT", "EXTINCT_IN_THE_WILD", "REGIONALLY_EXTINCT", "CRITICALLY_ENDANGERED", "ENDANGERED",
+        "VULNERABLE", "NEAR_THREATENED", "LEAST_CONCERN", "DATA_DEFICIENT", "NOT_APPLICABLE", "NOT_EVALUATED"
+     ]
+  },
   {
     "name" : "Nomenclature",
     "namespace" : "org.gbif.pipelines.io.avro",
@@ -145,7 +154,8 @@
         ]
       },
       {"name" : "usageParsedName", "type" : ["null", "ParsedName"], "default" : null},
-      {"name": "issues", "type": "IssueRecord", "default":{}}
+      {"name": "issues", "type": "IssueRecord", "default":{}},
+      {"name" : "iucnRedListCategory", "type" : ["null", "ThreatStatus"], "default" : null}
     ]
   },
 {

diff --git a/sdks/models/src/main/avro/occurrence-hdfs-record.avsc b/sdks/models/src/main/avro/occurrence-hdfs-record.avsc
@@ -1472,5 +1472,8 @@
       "type" : "array",
       "items" : [ "string", "null" ]
     }, "null" ]
-  } ]
+  }, {
+    "name" : "iucnRedListCategory",
+    "type" : [ "string", "null" ]
+  }]
 }