#114 tool to migrate vocabs across envs

gbif · Feb 20, 2024 · 22d6805 · 22d6805
1 parent b443d6c
commit 22d6805
Show file tree

Hide file tree

Showing 6 changed files with 318 additions and 70 deletions.
diff --git a/core/src/main/resources/liquibase/008-labels-definition-tables.xml b/core/src/main/resources/liquibase/008-labels-definition-tables.xml
@@ -206,4 +206,4 @@
       ]]>
     </sql>
   </changeSet>
-</databaseChangeLog>
+</databaseChangeLog>
diff --git a/model/src/main/java/org/gbif/vocabulary/model/Tag.java b/model/src/main/java/org/gbif/vocabulary/model/Tag.java
@@ -39,6 +39,12 @@ public class Tag implements Serializable, LenientEquals<Tag> {
   private LocalDateTime modified;
   private String modifiedBy;
 
+  public static Tag of(String name) {
+    Tag tag = new Tag();
+    tag.setName(name);
+    return tag;
+  }
+
   @Override
   public boolean lenientEquals(Tag other) {
     if (this == other) {

diff --git a/vocabulary-importer/README.md b/vocabulary-importer/README.md
@@ -149,3 +149,21 @@ If there were issues during the import the tool creates a file called `errors_{t
 
 Most of the issues should be related with duplicates that cannot be imported. Some of the restrictions are documented in the [core module](https://github.com/gbif/vocabulary/blob/master/core/notes.md).
 
+### How to migrate a vocabulary across environments
+
+The importer tool can also be used to migrate vocabularies across environments. To do so we just need to set the `migration` flag and specify the target API
+where the vocabulary has to be copied to, e.g.:
+
+```
+java -jar vocabulary-importer/target/vocabulary-importer-0.53.jar \
+--vocabularyName LifeStage \
+--migration \
+--apiUrl https://api.gbif-dev.org/v1/ \
+--targetApiUrl https://api.gbif-uat.org/v1/ \
+--apiUser myusername \
+--targetApiUser myusername \
+--apiPassword \
+--targetApiPassword
+```
+
+Note that you will be prompted to enter 2 passwords for each of the users.
diff --git a/vocabulary-importer/src/main/java/org/gbif/vocabulary/importer/CliApp.java b/vocabulary-importer/src/main/java/org/gbif/vocabulary/importer/CliApp.java
@@ -73,46 +73,79 @@ public static void main(String[] args) {
     VocabularyImporter vocabularyImporter =
         new VocabularyImporter(vocabularyClient, conceptClient, tagClient);
 
-    Path hiddenLabelsPath = null;
-    if (cliArgs.getHiddenLabelsPath() != null) {
-      hiddenLabelsPath = Paths.get(cliArgs.getHiddenLabelsPath());
-
-      if (!Files.exists(hiddenLabelsPath)) {
-        throw new IllegalArgumentException(
-            "Hidden labels path " + hiddenLabelsPath + " doesn't exist");
-      }
-    }
-
-    if (cliArgs.getCsvDelimiter() == null) {
-      throw new IllegalArgumentException("CSV delimiter is required");
-    }
-
-    if (cliArgs.importHiddenLabelsOnly) {
-      log.info("Calling the hidden labels importer");
-      vocabularyImporter.importHiddenLabels(
-          cliArgs.getCsvDelimiter().charAt(0),
+    if (cliArgs.isMigration()) {
+      VocabularyClient targetVocabularyClient =
+          clientBuilder
+              .withUrl(cliArgs.getTargetApiUrl())
+              .withExponentialBackoffRetry(Duration.ofMillis(1000), 2, 4)
+              .withCredentials(cliArgs.getTargetApiUser(), cliArgs.getTargetApiPassword())
+              .withObjectMapper(objectMapper)
+              .build(VocabularyClient.class);
+      ConceptClient targetConceptClient =
+          clientBuilder
+              .withUrl(cliArgs.getTargetApiUrl())
+              .withExponentialBackoffRetry(Duration.ofMillis(1000), 2, 4)
+              .withCredentials(cliArgs.getTargetApiUser(), cliArgs.getTargetApiPassword())
+              .withObjectMapper(objectMapper)
+              .build(ConceptClient.class);
+      TagClient targetTagClient =
+          clientBuilder
+              .withUrl(cliArgs.getTargetApiUrl())
+              .withExponentialBackoffRetry(Duration.ofMillis(1000), 2, 4)
+              .withCredentials(cliArgs.getTargetApiUser(), cliArgs.getTargetApiPassword())
+              .withObjectMapper(objectMapper)
+              .build(TagClient.class);
+
+      log.info("Calling the importer to migrate the vocabulary");
+      vocabularyImporter.migrateVocabulary(
           cliArgs.getVocabularyName(),
-          hiddenLabelsPath,
-          parseEncoding(cliArgs.encoding));
+          targetVocabularyClient,
+          targetConceptClient,
+          targetTagClient);
+
+      log.info("Migration done");
     } else {
-      Path conceptsPath = Paths.get(cliArgs.getConceptsPath());
+      Path hiddenLabelsPath = null;
+      if (cliArgs.getHiddenLabelsPath() != null) {
+        hiddenLabelsPath = Paths.get(cliArgs.getHiddenLabelsPath());
+
+        if (!Files.exists(hiddenLabelsPath)) {
+          throw new IllegalArgumentException(
+              "Hidden labels path " + hiddenLabelsPath + " doesn't exist");
+        }
+      }
 
-      if (!Files.exists(conceptsPath)) {
-        throw new IllegalArgumentException("Concepts path " + conceptsPath + " doesn't exist");
+      if (cliArgs.getCsvDelimiter() == null) {
+        throw new IllegalArgumentException("CSV delimiter is required");
       }
 
-      log.info("Calling the importer");
-      vocabularyImporter.importVocabulary(
-          cliArgs.getCsvDelimiter().charAt(0),
-          cliArgs.getListDelimiter(),
-          cliArgs.getVocabularyName(),
-          cliArgs.getVocabularyLabelEN(),
-          cliArgs.getVocabularyDefinitionEN(),
-          conceptsPath,
-          hiddenLabelsPath,
-          parseEncoding(cliArgs.encoding));
+      if (cliArgs.importHiddenLabelsOnly) {
+        log.info("Calling the hidden labels importer");
+        vocabularyImporter.importHiddenLabels(
+            cliArgs.getCsvDelimiter().charAt(0),
+            cliArgs.getVocabularyName(),
+            hiddenLabelsPath,
+            parseEncoding(cliArgs.encoding));
+      } else {
+        Path conceptsPath = Paths.get(cliArgs.getConceptsPath());
+
+        if (!Files.exists(conceptsPath)) {
+          throw new IllegalArgumentException("Concepts path " + conceptsPath + " doesn't exist");
+        }
+
+        log.info("Calling the importer");
+        vocabularyImporter.importVocabulary(
+            cliArgs.getCsvDelimiter().charAt(0),
+            cliArgs.getListDelimiter(),
+            cliArgs.getVocabularyName(),
+            cliArgs.getVocabularyLabelEN(),
+            cliArgs.getVocabularyDefinitionEN(),
+            conceptsPath,
+            hiddenLabelsPath,
+            parseEncoding(cliArgs.encoding));
+      }
+      log.info("Import done");
     }
-    log.info("Import done");
   }
 
   private static Charset parseEncoding(String encoding) {
@@ -139,17 +172,33 @@ public static class CliArgs {
         required = true)
     private String apiUrl;
 
+    @Parameter(
+        names = {"--targetApiUrl", "-ta"},
+        required = true)
+    private String targetApiUrl;
+
     @Parameter(
         names = {"--apiUser", "-au"},
         required = true)
     private String apiUser;
 
+    @Parameter(
+        names = {"--targetApiUser", "-tau"},
+        required = true)
+    private String targetApiUser;
+
     @Parameter(
         names = {"--apiPassword", "-ap"},
         required = true,
         password = true)
     private String apiPassword;
 
+    @Parameter(
+        names = {"--targetApiPassword", "-tap"},
+        required = true,
+        password = true)
+    private String targetApiPassword;
+
     @Parameter(
         names = {"--vocabularyName", "-vn"},
         required = true)
@@ -172,5 +221,8 @@ public static class CliArgs {
 
     @Parameter(names = {"--importHiddenLabelsOnly", "-hlo"})
     private boolean importHiddenLabelsOnly;
+
+    @Parameter(names = {"--migration", "-mi"})
+    private boolean migration;
   }
 }