DT-1194: Speed up integration tests by running them in parallel

build.gradle

@@ -484,6 +484,7 @@ tasks.register('testIntegration', Test) {
     }
     maxParallelForks = 8
     outputs.upToDateWhen { false }
+    systemProperty 'junit.jupiter.execution.parallel.enabled', 'true'
 }
 
 tasks.register('testUnit', Test) {

scripts/compose.yaml

@@ -20,6 +20,7 @@ services:
       - GOOGLE_SA_CERT
       - RBS_POOLID
       - RBS_INSTANCEURL
+      - TDR_LOG_APPENDER
     ports:
       - "8080:8080"
     volumes:

scripts/run

@@ -30,6 +30,9 @@ source "$(dirname "$0")"/init.sh
 die() { log_error "$*" >&2; echo ""; usage; exit 2; }  # complain to STDERR and exit with error
 needs_arg() { if [ -z "$OPTARG" ]; then die "No arg for --$OPT option"; fi; }
 
+# output plain logs instead of json
+export TDR_LOG_APPENDER=Console-Standard
+
 while getopts h-: OPT; do
   # support long options: https://stackoverflow.com/a/28466267/519360
   if [ "$OPT" = "-" ]; then   # long option: reformulate OPT and OPTARG
@@ -51,17 +54,24 @@ RUNTYPE=${1}
 
 run_tests() {
   cd "${ROOT_DIR}"
+  eval "$("${SCRIPTS_DIR}"/render-configs.sh -e)"
+
   # shellcheck disable=SC2086 # The array syntax and quoting seems to break gradle
   ./gradlew testUnit ${GRADLE_ARGS}
 
 #./gradlew --build-cache srcclr
 }
 
+wait_until_server_up() {
+  timeout 60 bash -c "until curl -s http:/localhost:8080/status; do sleep 1; done"
+}
+
 run_integration_tests() {
   TEST_ENV=${TEST_ENV:-local}
 
   cd "${ROOT_DIR}"
   start_docker integration
+  wait_until_server_up
   # shellcheck disable=SC2086 # The array syntax and quoting seems to break gradle
   ./gradlew testIntegration ${GRADLE_ARGS}
 
@@ -75,6 +85,8 @@ run_connected_tests() {
   TEST_ENV=${TEST_ENV:-local}
 
   cd "${ROOT_DIR}"
+  eval "$("${SCRIPTS_DIR}"/render-configs.sh -e)"
+
   # shellcheck disable=SC2086 # The array syntax and quoting seems to break gradle
   ./gradlew testConnected ${GRADLE_ARGS}
 

scripts/run-db

@@ -4,7 +4,7 @@
 usage() {
   cat <<-'EOF'
 usage: run-db [--clean-db] [-h|--help] COMMAND
-[--clean-db]       clean the database
+[--clean-db]       clean the database (only with `stop` or `start`)
 [-h|--help]        print this help text
 COMMAND is one of:
 - start            start a local database server
@@ -34,8 +34,6 @@ while getopts hc-: OPT; do  # allow -h and -- "with arg"
   # support long options: https://stackoverflow.com/a/28466267/519360
   if [ "$OPT" = "-" ]; then   # long option: reformulate OPT and OPTARG
     OPT="${OPTARG%%=*}"       # extract long option name
-    OPTARG="${OPTARG#"$OPT"}"   # extract long option argument (may be empty)
-    OPTARG="${OPTARG#=}"      # if long option argument, remove assigning `=`
   fi
   case "$OPT" in
     c | clean-db ) clean_db=true ;;
@@ -53,7 +51,7 @@ COMMAND=$1
 remove_database() {
   if [ "$clean_db" = "true" ]; then
     log_info "erasing existing database state"
-    rm -r "$TMP_DIR"/postgres-data
+    rm -r "$ROOT_DIR"/build/tmp/postgres-data
   fi
 }
 

src/main/java/bio/terra/service/dataset/flight/create/CreateDatasetGetOrCreateBucketStep.java

@@ -25,6 +25,7 @@
 import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.transaction.TransactionSystemException;
 
 public class CreateDatasetGetOrCreateBucketStep implements Step {
   private static final Logger logger =
@@ -79,8 +80,8 @@ public StepResult doStep(FlightContext context) throws InterruptedException {
               googleProjectResource.getServiceAccount());
 
       workingMap.put(FileMapKeys.BUCKET_INFO, bucketForFile);
-    } catch (BucketLockException blEx) {
-      return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, blEx);
+    } catch (BucketLockException | TransactionSystemException e) {
+      return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
     } catch (GoogleResourceNamingException ex) {
       return new StepResult(StepStatus.STEP_RESULT_FAILURE_FATAL, ex);
     }

src/main/java/bio/terra/service/dataset/flight/create/CreateDatasetGetOrCreateStorageAccountStep.java

@@ -12,9 +12,12 @@
 import bio.terra.stairway.FlightMap;
 import bio.terra.stairway.Step;
 import bio.terra.stairway.StepResult;
+import bio.terra.stairway.StepStatus;
 import java.util.UUID;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.dao.TransientDataAccessException;
+import org.springframework.transaction.TransactionSystemException;
 
 public class CreateDatasetGetOrCreateStorageAccountStep implements Step {
   private static Logger logger =
@@ -40,11 +43,16 @@ public StepResult doStep(FlightContext context) throws InterruptedException {
         workingMap.get(ProfileMapKeys.PROFILE_MODEL, BillingProfileModel.class);
     UUID datasetId = workingMap.get(DatasetWorkingMapKeys.DATASET_ID, UUID.class);
 
-    AzureStorageAccountResource storageAccount =
-        resourceService.getOrCreateDatasetStorageAccount(
-            DatasetJsonConversion.datasetRequestToDataset(datasetRequestModel, datasetId),
-            profileModel,
-            context.getFlightId());
+    AzureStorageAccountResource storageAccount;
+    try {
+      storageAccount =
+          resourceService.getOrCreateDatasetStorageAccount(
+              DatasetJsonConversion.datasetRequestToDataset(datasetRequestModel, datasetId),
+              profileModel,
+              context.getFlightId());
+    } catch (TransientDataAccessException | TransactionSystemException e) {
+      return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
+    }
 
     logger.info("Enabling Azure storage account logging");
     // Log files will reside in the storage account's $logs container

src/main/java/bio/terra/service/dataset/flight/create/DatasetCreateFlight.java

@@ -112,7 +112,8 @@ public DatasetCreateFlight(FlightMap inputParameters, Object applicationContext)
     if (platform.isAzure()) {
       addStep(
           new CreateDatasetGetOrCreateStorageAccountStep(
-              resourceService, datasetRequest, azureBlobStorePdao));
+              resourceService, datasetRequest, azureBlobStorePdao),
+          getDefaultRandomBackoffRetryRule(appConfig.getMaxStairwayThreads()));
 
       // Create the top level container
       addStep(
@@ -130,7 +131,7 @@ public DatasetCreateFlight(FlightMap inputParameters, Object applicationContext)
     // Create dataset metadata objects in postgres and lock the dataset
     addStep(
         new CreateDatasetMetadataStep(datasetDao, datasetRequest),
-        getDefaultExponentialBackoffRetryRule());
+        getDefaultRandomBackoffRetryRule(appConfig.getMaxStairwayThreads()));
 
     // For azure backed datasets, add a link co connect the storage account to the dataset
     if (platform.isAzure()) {

src/main/java/bio/terra/service/snapshot/flight/delete/DeleteSnapshotMarkProjectStep.java

@@ -7,11 +7,12 @@
 import bio.terra.stairway.FlightMap;
 import bio.terra.stairway.Step;
 import bio.terra.stairway.StepResult;
+import bio.terra.stairway.StepStatus;
 import bio.terra.stairway.exception.RetryException;
 import java.util.List;
 import java.util.UUID;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.springframework.dao.TransientDataAccessException;
+import org.springframework.transaction.TransactionSystemException;
 
 public class DeleteSnapshotMarkProjectStep implements Step {
 
@@ -26,19 +27,21 @@ public DeleteSnapshotMarkProjectStep(
     this.snapshotService = snapshotService;
   }
 
-  private static final Logger logger = LoggerFactory.getLogger(DeleteSnapshotMarkProjectStep.class);
-
   @Override
   public StepResult doStep(FlightContext context) throws InterruptedException, RetryException {
     FlightMap workingMap = context.getWorkingMap();
     UUID projectId = workingMap.get(SnapshotWorkingMapKeys.PROJECT_RESOURCE_ID, UUID.class);
 
-    List<UUID> projectsToBeDeleted =
-        resourceService.markUnusedProjectsForDelete(List.of(projectId));
+    try {
+      List<UUID> projectsToBeDeleted =
+          resourceService.markUnusedProjectsForDelete(List.of(projectId));
 
-    workingMap.put(SnapshotWorkingMapKeys.PROJECTS_MARKED_FOR_DELETE, projectsToBeDeleted);
+      workingMap.put(SnapshotWorkingMapKeys.PROJECTS_MARKED_FOR_DELETE, projectsToBeDeleted);
 
-    return StepResult.getStepResultSuccess();
+      return StepResult.getStepResultSuccess();
+    } catch (TransientDataAccessException | TransactionSystemException e) {
+      return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
+    }
   }
 
   @Override

src/test/java/bio/terra/app/controller/RepositoryApiControllerAccessTest.java

@@ -5,14 +5,16 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.is;
 
+import bio.terra.common.auth.Users;
 import bio.terra.common.category.Integration;
+import bio.terra.common.configuration.TestConfiguration.User;
 import bio.terra.integration.DataRepoFixtures;
 import bio.terra.integration.IntegrationTestConfiguration;
-import bio.terra.integration.UsersBase;
 import bio.terra.model.ConfigGroupModel;
 import bio.terra.model.ConfigModel;
 import bio.terra.model.ConfigParameterModel;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -30,14 +32,27 @@
 @SpringBootTest(classes = IntegrationTestConfiguration.class)
 @ActiveProfiles({"google", "integrationtest"})
 @Tag(Integration.TAG)
-class RepositoryApiControllerAccessTest extends UsersBase {
+@Disabled
+class RepositoryApiControllerAccessTest {
 
   @Autowired private DataRepoFixtures dataRepoFixtures;
+  @Autowired private Users users;
+
+  private User admin;
+  private User reader;
+
+  private User admin() {
+    return admin;
+  }
+
+  private User reader() {
+    return reader;
+  }
 
   @BeforeEach
-  @Override
   public void setup() throws Exception {
-    super.setup();
+    admin = users.admin();
+    reader = users.reader();
   }
 
   @Test

src/test/java/bio/terra/common/auth/Users.java

@@ -2,49 +2,38 @@
 
 import bio.terra.common.configuration.TestConfiguration;
 import bio.terra.common.configuration.TestConfiguration.User;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.stream.Collectors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Profile;
 import org.springframework.stereotype.Component;
 
 @Component
+@Profile("integrationtest")
 public class Users {
+  private static final Logger logger = LoggerFactory.getLogger(Users.class);
 
-  private Map<String, List<TestConfiguration.User>> usersByRole = new HashMap<>();
-  private Map<String, TestConfiguration.User> userByName = new HashMap<>();
+  private static final String ADMIN_ROLE = "admin";
+  private static final String STEWARD_ROLE = "steward";
+  private static final String CUSTODIAN_ROLE = "custodian";
+  private static final String READER_ROLE = "reader";
+  private static final String DISCOVERER_ROLE = "discoverer";
 
-  @Autowired
-  public Users(TestConfiguration testConfig) {
-    buildUsersByRole(testConfig.getUsers());
-  }
+  private final Map<String, List<User>> usersByRole;
 
-  private void buildUsersByRole(List<TestConfiguration.User> users) {
-    users.forEach(
-        user -> {
-          String role = user.getRole();
-          List<TestConfiguration.User> newList = new ArrayList<>();
-
-          if (usersByRole.containsKey(role)) newList = usersByRole.get(role);
-          newList.add(user);
-          usersByRole.put(role, newList);
-          userByName.put(user.getName(), user);
-        });
-  }
+  public record TestUsers(User admin, User steward, User custodian, User reader, User discoverer) {}
 
-  public TestConfiguration.User getUserForRole(String role) {
-    return getUserForRole(role, true);
-  }
-
-  public TestConfiguration.User getUserForRole(String role, boolean shuffle) {
-    return getUsersForRole(role, 1, shuffle).get(0);
+  @Autowired
+  public Users(TestConfiguration testConfig) {
+    usersByRole = testConfig.getUsers().stream().collect(Collectors.groupingBy(User::getRole));
   }
 
-  public TestConfiguration.User getUserForRole(String name, String role) {
-    List<User> usersForRole = getUsersForRole(role, -1, false);
+  private User getUserForRole(String name, String role) {
+    var usersForRole = usersByRole.get(role);
     return usersForRole.stream()
         .filter(u -> u.getName().equals(name))
         .findFirst()
@@ -60,24 +49,60 @@ public TestConfiguration.User getUserForRole(String name, String role) {
                             .collect(Collectors.joining(", ")))));
   }
 
-  public TestConfiguration.User getUser(String name) {
-    return userByName.get(name);
+  private User getUserForRole(String role) {
+    var users = usersByRole.get(role);
+    return users.get(new Random().nextInt(users.size()));
+  }
+
+  public User admin() {
+    return getUserForRole(ADMIN_ROLE);
+  }
+
+  public User admin(String name) {
+    return getUserForRole(name, ADMIN_ROLE);
+  }
+
+  public User steward() {
+    return getUserForRole(STEWARD_ROLE);
+  }
+
+  public User steward(String name) {
+    return getUserForRole(name, STEWARD_ROLE);
+  }
+
+  public User custodian() {
+    return getUserForRole(CUSTODIAN_ROLE);
+  }
+
+  public User custodian(String name) {
+    return getUserForRole(name, CUSTODIAN_ROLE);
+  }
+
+  public User reader() {
+    return getUserForRole(READER_ROLE);
+  }
+
+  public User reader(String name) {
+    return getUserForRole(name, READER_ROLE);
+  }
+
+  public User discoverer() {
+    return getUserForRole(DISCOVERER_ROLE);
+  }
+
+  public User discoverer(String name) {
+    return getUserForRole(name, DISCOVERER_ROLE);
   }
 
-  public List<TestConfiguration.User> getUsersForRole(String role, int numUsers, boolean shuffle) {
-    if (role == null) {
-      throw new RuntimeException("Role not specified");
-    }
-    List<TestConfiguration.User> usersList = usersByRole.get(role);
-    if (numUsers != -1 && usersList.size() < numUsers) {
-      throw new RuntimeException("not enough users for " + role);
-    }
-    if (shuffle) {
-      Collections.shuffle(usersList);
-    }
-    if (numUsers != -1) {
-      return usersList.subList(0, numUsers);
-    }
-    return usersList;
+  public TestUsers testUsers() {
+    TestUsers testUsers = new TestUsers(admin(), steward(), custodian(), reader(), discoverer());
+    logger.info(
+        "admin: {}; steward: {}; custodian: {}; reader: {}; discoverer: {}",
+        testUsers.admin().getName(),
+        testUsers.steward().getName(),
+        testUsers.custodian().getName(),
+        testUsers.reader().getName(),
+        testUsers.discoverer().getName());
+    return testUsers;
   }
 }