DT-1194: Speed up integration tests by running them in parallel

build.gradle

@@ -484,6 +484,7 @@ tasks.register('testIntegration', Test) {
     }
     maxParallelForks = 8
     outputs.upToDateWhen { false }
+    systemProperty 'junit.jupiter.execution.parallel.enabled', 'true'
 }
 
 tasks.register('testUnit', Test) {

scripts/compose.yaml

@@ -20,6 +20,7 @@ services:
       - GOOGLE_SA_CERT
       - RBS_POOLID
       - RBS_INSTANCEURL
+      - TDR_LOG_APPENDER
     ports:
       - "8080:8080"
     volumes:

scripts/run

@@ -30,6 +30,9 @@ source "$(dirname "$0")"/init.sh
 die() { log_error "$*" >&2; echo ""; usage; exit 2; }  # complain to STDERR and exit with error
 needs_arg() { if [ -z "$OPTARG" ]; then die "No arg for --$OPT option"; fi; }
 
+# output plain logs instead of json
+export TDR_LOG_APPENDER=Console-Standard
+
 while getopts h-: OPT; do
   # support long options: https://stackoverflow.com/a/28466267/519360
   if [ "$OPT" = "-" ]; then   # long option: reformulate OPT and OPTARG
@@ -57,11 +60,16 @@ run_tests() {
 #./gradlew --build-cache srcclr
 }
 
+wait_until_server_up() {
+  timeout 60 bash -c "until curl -s http:/localhost:8080/status; do sleep 1; done"
+}
+
 run_integration_tests() {
   TEST_ENV=${TEST_ENV:-local}
 
   cd "${ROOT_DIR}"
   start_docker integration
+  wait_until_server_up
   # shellcheck disable=SC2086 # The array syntax and quoting seems to break gradle
   ./gradlew testIntegration ${GRADLE_ARGS}
 

src/main/java/bio/terra/service/dataset/flight/create/CreateDatasetGetOrCreateBucketStep.java

@@ -25,6 +25,7 @@
 import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.transaction.TransactionSystemException;
 
 public class CreateDatasetGetOrCreateBucketStep implements Step {
   private static final Logger logger =
@@ -79,8 +80,8 @@ public StepResult doStep(FlightContext context) throws InterruptedException {
               googleProjectResource.getServiceAccount());
 
       workingMap.put(FileMapKeys.BUCKET_INFO, bucketForFile);
-    } catch (BucketLockException blEx) {
-      return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, blEx);
+    } catch (BucketLockException | TransactionSystemException e) {
+      return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
     } catch (GoogleResourceNamingException ex) {
       return new StepResult(StepStatus.STEP_RESULT_FAILURE_FATAL, ex);
     }

src/main/java/bio/terra/service/dataset/flight/create/CreateDatasetGetOrCreateStorageAccountStep.java

@@ -12,9 +12,12 @@
 import bio.terra.stairway.FlightMap;
 import bio.terra.stairway.Step;
 import bio.terra.stairway.StepResult;
+import bio.terra.stairway.StepStatus;
 import java.util.UUID;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.dao.TransientDataAccessException;
+import org.springframework.transaction.TransactionSystemException;
 
 public class CreateDatasetGetOrCreateStorageAccountStep implements Step {
   private static Logger logger =
@@ -40,11 +43,16 @@ public StepResult doStep(FlightContext context) throws InterruptedException {
         workingMap.get(ProfileMapKeys.PROFILE_MODEL, BillingProfileModel.class);
     UUID datasetId = workingMap.get(DatasetWorkingMapKeys.DATASET_ID, UUID.class);
 
-    AzureStorageAccountResource storageAccount =
-        resourceService.getOrCreateDatasetStorageAccount(
-            DatasetJsonConversion.datasetRequestToDataset(datasetRequestModel, datasetId),
-            profileModel,
-            context.getFlightId());
+    AzureStorageAccountResource storageAccount;
+    try {
+      storageAccount =
+          resourceService.getOrCreateDatasetStorageAccount(
+              DatasetJsonConversion.datasetRequestToDataset(datasetRequestModel, datasetId),
+              profileModel,
+              context.getFlightId());
+    } catch (TransientDataAccessException | TransactionSystemException e) {
+      return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
+    }
 
     logger.info("Enabling Azure storage account logging");
     // Log files will reside in the storage account's $logs container

src/main/java/bio/terra/service/dataset/flight/create/DatasetCreateFlight.java

@@ -112,7 +112,8 @@ public DatasetCreateFlight(FlightMap inputParameters, Object applicationContext)
     if (platform.isAzure()) {
       addStep(
           new CreateDatasetGetOrCreateStorageAccountStep(
-              resourceService, datasetRequest, azureBlobStorePdao));
+              resourceService, datasetRequest, azureBlobStorePdao),
+          getDefaultRandomBackoffRetryRule(appConfig.getMaxStairwayThreads()));
 
       // Create the top level container
       addStep(
@@ -130,7 +131,7 @@ public DatasetCreateFlight(FlightMap inputParameters, Object applicationContext)
     // Create dataset metadata objects in postgres and lock the dataset
     addStep(
         new CreateDatasetMetadataStep(datasetDao, datasetRequest),
-        getDefaultExponentialBackoffRetryRule());
+        getDefaultRandomBackoffRetryRule(appConfig.getMaxStairwayThreads()));
 
     // For azure backed datasets, add a link co connect the storage account to the dataset
     if (platform.isAzure()) {

src/main/java/bio/terra/service/resourcemanagement/azure/AzureResourceDao.java

@@ -111,6 +111,21 @@ public AzureResourceDao(NamedParameterJdbcTemplate jdbcTemplate) throws SQLExcep
 
   @Transactional(propagation = Propagation.REQUIRED, isolation = Isolation.SERIALIZABLE)
   public UUID createApplicationDeployment(AzureApplicationDeploymentResource application) {
+    try {
+      var resource =
+          retrieveApplicationDeploymentByName(application.getAzureApplicationDeploymentName());
+      if (resource != null) {
+        // The database requires azure_application_deployment_name to be unique
+        // (see retrieveApplicationDeploymentBy below). To enforce that, we should add a db
+        // constraint
+        // and use ON CONFLICT to return the existing record instead. For now, check for a duplicate
+        // inside this transaction before writing a new record.
+        return resource.getId();
+      }
+    } catch (AzureResourceNotFoundException e) {
+      // This is the expected case, where the application deployment does not exist.
+    }
+
     String sql =
         "INSERT INTO application_deployment_resource "
             + "(azure_application_deployment_id,azure_application_deployment_name,azure_resource_group_name,"

src/main/java/bio/terra/service/snapshot/flight/delete/DeleteSnapshotMarkProjectStep.java

@@ -7,11 +7,12 @@
 import bio.terra.stairway.FlightMap;
 import bio.terra.stairway.Step;
 import bio.terra.stairway.StepResult;
+import bio.terra.stairway.StepStatus;
 import bio.terra.stairway.exception.RetryException;
 import java.util.List;
 import java.util.UUID;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.springframework.dao.TransientDataAccessException;
+import org.springframework.transaction.TransactionSystemException;
 
 public class DeleteSnapshotMarkProjectStep implements Step {
 
@@ -26,19 +27,21 @@ public DeleteSnapshotMarkProjectStep(
     this.snapshotService = snapshotService;
   }
 
-  private static final Logger logger = LoggerFactory.getLogger(DeleteSnapshotMarkProjectStep.class);
-
   @Override
   public StepResult doStep(FlightContext context) throws InterruptedException, RetryException {
     FlightMap workingMap = context.getWorkingMap();
     UUID projectId = workingMap.get(SnapshotWorkingMapKeys.PROJECT_RESOURCE_ID, UUID.class);
 
-    List<UUID> projectsToBeDeleted =
-        resourceService.markUnusedProjectsForDelete(List.of(projectId));
+    try {
+      List<UUID> projectsToBeDeleted =
+          resourceService.markUnusedProjectsForDelete(List.of(projectId));
 
-    workingMap.put(SnapshotWorkingMapKeys.PROJECTS_MARKED_FOR_DELETE, projectsToBeDeleted);
+      workingMap.put(SnapshotWorkingMapKeys.PROJECTS_MARKED_FOR_DELETE, projectsToBeDeleted);
 
-    return StepResult.getStepResultSuccess();
+      return StepResult.getStepResultSuccess();
+    } catch (TransientDataAccessException | TransactionSystemException e) {
+      return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
+    }
   }
 
   @Override

src/test/java/bio/terra/app/controller/RepositoryApiControllerAccessTest.java

@@ -6,13 +6,15 @@
 import static org.hamcrest.Matchers.is;
 
 import bio.terra.common.category.Integration;
+import bio.terra.common.configuration.TestConfiguration.User;
 import bio.terra.integration.DataRepoFixtures;
 import bio.terra.integration.IntegrationTestConfiguration;
-import bio.terra.integration.UsersBase;
+import bio.terra.integration.Users;
 import bio.terra.model.ConfigGroupModel;
 import bio.terra.model.ConfigModel;
 import bio.terra.model.ConfigParameterModel;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -30,14 +32,27 @@
 @SpringBootTest(classes = IntegrationTestConfiguration.class)
 @ActiveProfiles({"google", "integrationtest"})
 @Tag(Integration.TAG)
-class RepositoryApiControllerAccessTest extends UsersBase {
+@Disabled
+class RepositoryApiControllerAccessTest {
 
   @Autowired private DataRepoFixtures dataRepoFixtures;
+  @Autowired private Users users;
+
+  private User admin;
+  private User reader;
+
+  private User admin() {
+    return admin;
+  }
+
+  private User reader() {
+    return reader;
+  }
 
   @BeforeEach
-  @Override
   public void setup() throws Exception {
-    super.setup();
+    admin = users.admin();
+    reader = users.reader();
   }
 
   @Test

src/test/java/bio/terra/common/auth/AuthService.java

@@ -60,9 +60,9 @@ public class AuthService {
   public AuthService(TestConfiguration testConfig, IamProviderInterface iamProvider)
       throws Exception {
     this.testConfig = testConfig;
-    Optional<String> pemfilename = Optional.ofNullable(testConfig.getJadePemFileName());
+    Optional<String> pemfilename = Optional.ofNullable(testConfig.jadePemFileName());
     pemfilename.ifPresent(s -> pemfile = new File(s));
-    saEmail = testConfig.getJadeEmail();
+    saEmail = testConfig.jadeEmail();
     httpTransport = GoogleNetHttpTransport.newTrustedTransport();
     this.iamProvider = iamProvider;
   }
@@ -85,7 +85,7 @@ private GoogleCredential buildCredential(String email, List<String> scopes)
       throws IOException, GeneralSecurityException {
     if (pemfile == null) {
       throw new IllegalStateException(
-          String.format("pemfile not found: %s", testConfig.getJadePemFileName()));
+          String.format("pemfile not found: %s", testConfig.jadePemFileName()));
     }
     return new GoogleCredential.Builder()
         .setTransport(httpTransport)