Split main webservice module into core, import (using neo4j) and webs…

…ervices. That allows to reuse matching and other componnents without the drag of dropwizard and neo4j
CatalogueOfLife · Jan 30, 2025 · e998d53 · e998d53
1 parent 587e4a2
commit e998d53
Show file tree

Hide file tree

Showing 1,609 changed files with 1,714 additions and 1,299 deletions.
diff --git a/api/pom.xml b/api/pom.xml
@@ -13,7 +13,7 @@
 
   <artifactId>api</artifactId>
   <packaging>jar</packaging>
-  <name>COL API</name>
+  <name>CLB API</name>
 
   <properties>
     <jena.version>4.10.0</jena.version>

diff --git a/coldp/pom.xml b/coldp/pom.xml
@@ -10,7 +10,7 @@
   </parent>
 
   <artifactId>coldp</artifactId>
-  <name>COL Data Packages</name>
+  <name>CLB Data Packages</name>
   <description>This module provides terms for the ColDP format with minimal dependencies</description>
 
   <dependencies>

diff --git a/core/dataset/003/3/4-names.txt.gz b/core/dataset/003/3/4-names.txt.gz
diff --git a/core/dataset/003/3/4-tree.txt.gz b/core/dataset/003/3/4-tree.txt.gz
diff --git a/core/dataset/003/3/sector/2/1-names.txt.gz b/core/dataset/003/3/sector/2/1-names.txt.gz
diff --git a/core/dataset/003/3/sector/3/1-names.txt.gz b/core/dataset/003/3/sector/3/1-names.txt.gz
diff --git a/core/dataset/003/3/sector/4/1-names.txt.gz b/core/dataset/003/3/sector/4/1-names.txt.gz
diff --git a/core/pom.xml b/core/pom.xml
@@ -0,0 +1,330 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.catalogueoflife</groupId>
+    <artifactId>motherpom</artifactId>
+    <version>1.0-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>core</artifactId>
+  <name>CLB core</name>
+  <description>Core components of ChecklistBank not yet tight to any webservice framework</description>
+
+  <dependencies>
+    <!-- col & gbif -->
+    <dependency>
+      <groupId>org.catalogueoflife</groupId>
+      <artifactId>coldp</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.catalogueoflife</groupId>
+      <artifactId>metadata</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.catalogueoflife</groupId>
+      <artifactId>api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.gbif</groupId>
+      <artifactId>name-parser-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.catalogueoflife</groupId>
+      <artifactId>parser</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.catalogueoflife</groupId>
+      <artifactId>reader</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.catalogueoflife</groupId>
+      <artifactId>reader-xls</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.catalogueoflife</groupId>
+      <artifactId>dao</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.catalogueoflife</groupId>
+      <artifactId>doi</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.gbif</groupId>
+      <artifactId>dwc-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.gbif</groupId>
+      <artifactId>dwca-io</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.gbif</groupId>
+      <artifactId>text-tree</artifactId>
+    </dependency>
+    <!-- DB -->
+    <dependency>
+      <groupId>org.mybatis</groupId>
+      <artifactId>mybatis</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.postgresql</groupId>
+      <artifactId>postgresql</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.zaxxer</groupId>
+      <artifactId>HikariCP</artifactId>
+    </dependency>
+
+    <!-- mapdb & kryo -->
+    <!-- we use kryo to serialize data into binary which is > 10 times faster than jackson even with smile -->
+    <dependency>
+      <groupId>org.mapdb</groupId>
+      <artifactId>mapdb</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.esotericsoftware</groupId>
+      <artifactId>kryo</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.fury</groupId>
+      <artifactId>fury-core</artifactId>
+    </dependency>
+
+    <!-- Elasticsearch / Lucene -->
+    <dependency>
+      <groupId>org.elasticsearch.client</groupId>
+      <artifactId>elasticsearch-rest-client</artifactId>
+      <version>${elasticsearch.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.github.docker-java</groupId>
+      <artifactId>docker-java</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.github.docker-java</groupId>
+      <artifactId>docker-java-transport-httpclient5</artifactId>
+    </dependency>
+
+    <!-- collections & CSV -->
+    <dependency>
+      <groupId>it.unimi.dsi</groupId>
+      <artifactId>fastutil</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.univocity</groupId>
+      <artifactId>univocity-parsers</artifactId>
+    </dependency>
+
+    <!-- unicode, transliterations -->
+    <dependency>
+      <groupId>com.ibm.icu</groupId>
+      <artifactId>icu4j</artifactId>
+    </dependency>
+
+    <!-- Jackson -->
+    <dependency>
+      <groupId>jakarta.xml.bind</groupId>
+      <artifactId>jakarta.xml.bind-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-annotations</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.datatype</groupId>
+      <artifactId>jackson-datatype-jsr310</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-xml</artifactId>
+    </dependency>
+
+    <!-- commons -->
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-text</artifactId>
+    </dependency>
+
+    <!-- other -->
+    <dependency>
+      <groupId>org.simplejavamail</groupId>
+      <artifactId>simple-java-mail</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.github.ben-manes.caffeine</groupId>
+      <artifactId>caffeine</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.jsonwebtoken</groupId>
+      <artifactId>jjwt-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.jsonwebtoken</groupId>
+      <artifactId>jjwt-impl</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.jsonwebtoken</groupId>
+      <artifactId>jjwt-jackson</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tika</groupId>
+      <artifactId>tika-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.poi</groupId>
+      <artifactId>poi-ooxml</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-util</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>jakarta.ws.rs</groupId>
+      <artifactId>jakarta.ws.rs-api</artifactId>
+    </dependency>
+
+    <!-- logging -->
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>jul-to-slf4j</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>jakarta.validation</groupId>
+      <artifactId>jakarta.validation-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>co.elastic.logging</groupId>
+      <artifactId>logback-ecs-encoder</artifactId>
+    </dependency>
+
+    <!-- TEST SCOPE -->
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.vintage</groupId>
+      <artifactId>junit-vintage-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.assertj</groupId>
+      <artifactId>assertj-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.catalogueoflife</groupId>
+      <artifactId>api</artifactId>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.catalogueoflife</groupId>
+      <artifactId>dao</artifactId>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.javers</groupId>
+      <artifactId>javers-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>postgresql</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>elasticsearch</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <!-- come with dropwizard -->
+    <dependency>
+      <groupId>com.fasterxml.jackson.jakarta.rs</groupId>
+      <artifactId>jackson-jakarta-rs-json-provider</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.hibernate.validator</groupId>
+      <artifactId>hibernate-validator</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.glassfish</groupId>
+      <artifactId>jakarta.el</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <resources>
+      <resource>
+        <directory>src/main/resources</directory>
+        <filtering>false</filtering>
+        <excludes>
+          <exclude>**/*.yaml</exclude>
+        </excludes>
+      </resource>
+      <resource>
+        <directory>src/main/resources</directory>
+        <filtering>true</filtering>
+        <includes>
+          <include>**/*.yaml</include>
+        </includes>
+      </resource>
+    </resources>
+
+    <testResources>
+      <testResource>
+        <directory>src/test/resources</directory>
+        <filtering>false</filtering>
+        <excludes>
+          <exclude>gbifAuth.yaml</exclude>
+          <exclude>datacite.yaml</exclude>
+        </excludes>
+      </testResource>
+      <testResource>
+        <directory>src/test/resources</directory>
+        <filtering>true</filtering>
+        <includes>
+          <include>gbifAuth.yaml</include>
+          <include>datacite.yaml</include>
+        </includes>
+      </testResource>
+    </testResources>
+  </build>
+
+</project>
diff --git a/...life/catalogue/assembly/SectorDelete.java → ...life/catalogue/assembly/SectorDelete.java b/...life/catalogue/assembly/SectorDelete.java → ...life/catalogue/assembly/SectorDelete.java
diff --git a/.../catalogue/assembly/SectorDeleteFull.java → .../catalogue/assembly/SectorDeleteFull.java b/.../catalogue/assembly/SectorDeleteFull.java → .../catalogue/assembly/SectorDeleteFull.java
@@ -3,7 +3,6 @@
 import life.catalogue.api.model.DSID;
 import life.catalogue.api.model.DSIDValue;
 import life.catalogue.api.model.Sector;
-import life.catalogue.api.model.User;
 import life.catalogue.api.vocab.ImportState;
 import life.catalogue.dao.SectorDao;
 import life.catalogue.dao.SectorImportDao;

diff --git a/...fe/catalogue/assembly/SectorRunnable.java → ...fe/catalogue/assembly/SectorRunnable.java b/...fe/catalogue/assembly/SectorRunnable.java → ...fe/catalogue/assembly/SectorRunnable.java
@@ -7,6 +7,8 @@
 import life.catalogue.api.util.ObjectUtils;
 import life.catalogue.api.vocab.*;
 import life.catalogue.common.util.LoggingUtils;
+import life.catalogue.concurrent.DatasetBlockedException;
+import life.catalogue.concurrent.DatasetLock;
 import life.catalogue.dao.DatasetInfoCache;
 import life.catalogue.dao.SectorDao;
 import life.catalogue.dao.SectorImportDao;
@@ -39,6 +41,7 @@ abstract class SectorRunnable implements Runnable {
     Rank.SUBSPECIES, Rank.VARIETY, Rank.FORM
   );
 
+  protected final UUID key = UUID.randomUUID();
   protected final DSID<Integer> sectorKey;
   protected final int subjectDatasetKey;
   protected Sector sector;
@@ -126,9 +129,6 @@ public Map<String, EditorialDecision> getDecisions() {
 
   @Override
   public void run() {
-    LoggingUtils.setSectorMDC(sectorKey, state.getAttempt());
-    LoggingUtils.setSourceMDC(sector.getSubjectDatasetKey());
-
     try {
       state.setStarted(LocalDateTime.now());
       state.setState( ImportState.PREPARING);

diff --git a/...a/life/catalogue/assembly/SectorSync.java → ...a/life/catalogue/assembly/SectorSync.java b/...a/life/catalogue/assembly/SectorSync.java → ...a/life/catalogue/assembly/SectorSync.java
diff --git a/...ife/catalogue/assembly/SycnException.java → ...ife/catalogue/assembly/SycnException.java b/...ife/catalogue/assembly/SycnException.java → ...ife/catalogue/assembly/SycnException.java
diff --git a/.../life/catalogue/assembly/SyncFactory.java → .../life/catalogue/assembly/SyncFactory.java b/.../life/catalogue/assembly/SyncFactory.java → .../life/catalogue/assembly/SyncFactory.java
@@ -2,7 +2,6 @@
 
 import life.catalogue.api.model.DSID;
 import life.catalogue.api.model.Sector;
-import life.catalogue.api.vocab.DatasetOrigin;
 import life.catalogue.common.id.ShortUUID;
 import life.catalogue.dao.DatasetInfoCache;
 import life.catalogue.dao.EstimateDao;