Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ plugins {
id 'java'
id 'io.spring.dependency-management' version '1.1.6'
id 'jacoco'
id 'com.diffplug.spotless' version '6.7.1'
id 'com.diffplug.spotless' version '6.25.0'
id 'com.dorongold.task-tree' version '4.0.0'
// enables release info in sentry events
id 'com.gorylenko.gradle-git-properties' version '2.4.2'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,18 @@ public class ApplicationConfiguration {
private String dnsName;
private String resourceId;
private String userId;

/**
* Size of the Stairway thread pool. The pool is consumed by requests and by file load threads.
*/
private int maxStairwayThreads;

/** Maximum number of file loads allowed in the input array in a bulk file load */
private int maxBulkFileLoadArray;

/** Number of file loads to run concurrently in a bulk file load */
private int loadConcurrentFiles;

/**
* Number of file loads to run concurrently. NOTE: the maximum number of threads used for load is
* one for the driver flight and N for the number of concurrent files: {@code
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ public class JdbcConfiguration {
private String username;
private String password;
private String changesetFile;

/**
* Maximum number of database connections in the connection pool; -1 means no limit The goal of
* these parameters is to prevent waiting for a database connection.
Expand Down
1 change: 1 addition & 0 deletions src/main/java/bio/terra/common/LockOperation.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ public enum LockOperation {
LockOperation(boolean isLock) {
this(isLock, null, null);
}

/**
* @return additional guidance for users investigating a failed lock operation, usually as part of
* a failed job / flight.
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/bio/terra/common/SynapseColumn.java
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,8 @@ public static String translateDataType(
case TIME -> "time";
// Data of type RECORD contains table-like that can be nested or repeated
// It's provided in JSON format, making it hard to parse from inside a CSV/JSON ingest
case RECORD -> throw new NotSupportedException(
"RECORD type is not yet supported for synapse");
case RECORD ->
throw new NotSupportedException("RECORD type is not yet supported for synapse");
};
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,7 @@ void overwriteGroupPolicyEmailsIncludeRequestingUser(
String policyName,
List<String> emailAddresses)
throws InterruptedException;

/**
* @param accessToken valid oauth token for the account modifying the group policy members
* @param groupName name of Firecloud managed group
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ public class SamApiService {

private final SamConfiguration samConfig;
private final ConfigurationService configurationService;

/** OkHttpClients should be shared among requests to reduce latency and save memory * */
private final OkHttpClient sharedHttpClient;

Expand Down
10 changes: 6 additions & 4 deletions src/main/java/bio/terra/service/duos/DuosClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -121,10 +121,12 @@ public DuosDatasetApprovedUsers getApprovedUsers(String duosId) {
static ErrorReportException convertToDataRepoException(
HttpStatusCodeException duosEx, String duosId) {
return switch (HttpStatus.valueOf(duosEx.getStatusCode().value())) {
case BAD_REQUEST -> new DuosDatasetBadRequestException(
"DUOS dataset identifier %s is malformed".formatted(duosId), duosEx);
case NOT_FOUND -> new DuosDatasetNotFoundException(
"Could not find DUOS dataset for identifier %s".formatted(duosId), duosEx);
case BAD_REQUEST ->
new DuosDatasetBadRequestException(
"DUOS dataset identifier %s is malformed".formatted(duosId), duosEx);
case NOT_FOUND ->
new DuosDatasetNotFoundException(
"Could not find DUOS dataset for identifier %s".formatted(duosId), duosEx);
default -> new DuosInternalServerErrorException("Unexpected error from DUOS", duosEx);
};
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1462,16 +1462,16 @@ private Object extractValue(ResultSet resultSet, Column column) {
return switch (column.getType()) {
case BOOLEAN -> resultSet.getBoolean(column.getName());
case BYTES -> resultSet.getBytes(column.getName());
case DIRREF, FILEREF, STRING, TEXT, DATE, DATETIME, TIMESTAMP -> resultSet.getString(
column.getName());
case DIRREF, FILEREF, STRING, TEXT, DATE, DATETIME, TIMESTAMP ->
resultSet.getString(column.getName());
case FLOAT -> resultSet.getFloat(column.getName());
case FLOAT64 -> resultSet.getDouble(column.getName());
case INTEGER -> resultSet.getInt(column.getName());
case INT64 -> resultSet.getLong(column.getName());
case NUMERIC -> resultSet.getFloat(column.getName());
case TIME -> resultSet.getTime(column.getName());
default -> throw new IllegalArgumentException(
"Unknown datatype '" + column.getType() + "'");
default ->
throw new IllegalArgumentException("Unknown datatype '" + column.getType() + "'");
};
} catch (SQLException e) {
throw new PdaoException("Error reading data", e);
Expand All @@ -1492,17 +1492,11 @@ private Object extractValue(ResultSet resultSet, Column column) {
TypeReference<?> targetType =
switch (column.getType()) {
case BOOLEAN -> new TypeReference<List<Boolean>>() {};
case DATE,
DATETIME,
DIRREF,
FILEREF,
STRING,
TEXT,
TIME,
TIMESTAMP -> new TypeReference<List<String>>() {};
case DATE, DATETIME, DIRREF, FILEREF, STRING, TEXT, TIME, TIMESTAMP ->
new TypeReference<List<String>>() {};
case FLOAT, FLOAT64, INTEGER, INT64, NUMERIC -> new TypeReference<List<Number>>() {};
default -> throw new IllegalArgumentException(
"Unknown datatype '" + column.getType() + "'");
default ->
throw new IllegalArgumentException("Unknown datatype '" + column.getType() + "'");
};
try {
return objectMapper.readValue(rawValue, targetType);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@ private AzureConstants() {}

/** The code that the Azure resource management APIs found when a resource is not found */
public static final String RESOURCE_NOT_FOUND_CODE = "ResourceNotFound";

/**
* The code that the Azure security insights management APIs found when a resource is not found
*/
public static final String NOT_FOUND_CODE = "NotFound";

/**
* The code that the Azure resource management APIs found when an invalid request is sent. This
* can sometimes be equivalent to a 404
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,7 @@ public List<FireStoreDirectoryEntry> enumerateAll(Firestore firestore, String co
CollectionReference dirColl = firestore.collection(collectionId);
return query(dirColl.orderBy("path"));
}

// -- private methods --

List<FireStoreDirectoryEntry> enumerateDirectory(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ public void writeListToCloudFile(String path, List<String> contentsToWrite, Stri
writeStreamToCloudFile(path, stream, projectId);
}
}

/**
* Write a {@link Stream} to a GCS file separated by newlines
*
Expand Down
1 change: 1 addition & 0 deletions src/main/java/bio/terra/service/load/LoadService.java
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ public LoadHistoryIterator loadHistoryIterator(
List<BulkLoadHistoryModel> backingList, int chunkSize) {
return new LoadHistoryIterator(backingList, chunkSize);
}

/**
* A convenience class wrapping the getting of load history table rows in an Iterator. The
* Iterator's elements are a list of BulkLoadHistoryModel chunk of the full results retrieved from
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
@Service
public class PolicyApiService {
private final PolicyServiceConfiguration policyServiceConfiguration;

/** Clients should be shared among requests to reduce latency and save memory * */
private final Client sharedHttpClient;

Expand Down
13 changes: 7 additions & 6 deletions src/main/java/bio/terra/service/policy/PolicyService.java
Original file line number Diff line number Diff line change
Expand Up @@ -131,10 +131,11 @@ public void deletePaoIfExists(UUID resourceId) {
@VisibleForTesting
static RuntimeException convertApiException(ApiException ex) {
return switch (ex.getCode()) {
case HttpStatus.SC_UNAUTHORIZED -> new PolicyServiceAuthorizationException(
"Not authorized to access Terra Policy Service", ex.getCause());
case HttpStatus.SC_NOT_FOUND -> new PolicyServiceNotFoundException(
"Policy access object not found", ex);
case HttpStatus.SC_UNAUTHORIZED ->
new PolicyServiceAuthorizationException(
"Not authorized to access Terra Policy Service", ex.getCause());
case HttpStatus.SC_NOT_FOUND ->
new PolicyServiceNotFoundException("Policy access object not found", ex);
case HttpStatus.SC_BAD_REQUEST -> {
if (StringUtils.containsIgnoreCase(ex.getMessage(), "duplicate")) {
// TODO - this special handling may be removed once
Expand All @@ -146,8 +147,8 @@ yield new PolicyServiceDuplicateException(
yield new PolicyServiceApiException(ex);
}
}
case HttpStatus.SC_CONFLICT -> new PolicyConflictException(
"Policy access object already exists", ex);
case HttpStatus.SC_CONFLICT ->
new PolicyConflictException("Policy access object already exists", ex);
default -> new PolicyServiceApiException(ex);
};
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ public class BufferService {
private final ResourceBufferServiceConfiguration bufferServiceConfiguration;
private final GoogleResourceConfiguration googleConfig;
private final GoogleResourceManagerService googleResourceManagerService;

/** Clients should be shared among requests to reduce latency and save memory * */
private final Client sharedHttpClient;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ public AccessInfoModel accessInfoFromSnapshot(
final Snapshot snapshot, final AuthenticatedUserRequest userRequest) {
return accessInfoFromSnapshot(snapshot, userRequest, null);
}

/** Generate an {@link AccessInfoModel} from a Snapshot */
public AccessInfoModel accessInfoFromSnapshot(
final Snapshot snapshot, final AuthenticatedUserRequest userRequest, String forTable) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ public class DatasetProject {
private UUID profileId;
private String
dataProject; // Project id of the snapshot data project--which is a string and feels like a

// name

public UUID getId() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ private void validateSnapshotContents(
}
case BYQUERY -> validateSnapshotQuerySpec(contents.getQuerySpec(), errors);
case BYROWID -> validateSnapshotRowIdSpec(contents.getRowIdSpec(), errors);
case BYREQUESTID -> validateSnapshotRequestIdSpec(
contents.getRequestIdSpec(), errors);
case BYREQUESTID ->
validateSnapshotRequestIdSpec(contents.getRequestIdSpec(), errors);
}
});
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -219,18 +219,19 @@ public SnapshotCreateFlight(FlightMap inputParameters, Object applicationContext
new CreateSnapshotByAssetParquetFilesAzureStep(
azureSynapsePdao, snapshotService, snapshotReq, snapshotId)));
}
case BYFULLVIEW -> addStep(
platform.choose(
() ->
new CreateSnapshotPrimaryDataFullViewGcpStep(
bigQuerySnapshotPdao,
snapshotDao,
snapshotService,
snapshotReq,
sourceDataset),
() ->
new CreateSnapshotByFullViewParquetFilesAzureStep(
azureSynapsePdao, snapshotService, snapshotReq, snapshotId)));
case BYFULLVIEW ->
addStep(
platform.choose(
() ->
new CreateSnapshotPrimaryDataFullViewGcpStep(
bigQuerySnapshotPdao,
snapshotDao,
snapshotService,
snapshotReq,
sourceDataset),
() ->
new CreateSnapshotByFullViewParquetFilesAzureStep(
azureSynapsePdao, snapshotService, snapshotReq, snapshotId)));
case BYQUERY -> {
addStep(new CreateSnapshotValidateQueryStep(datasetService, snapshotReq));
addStep(
Expand All @@ -255,14 +256,15 @@ public SnapshotCreateFlight(FlightMap inputParameters, Object applicationContext
snapshotId,
sourceDataset)));
}
case BYROWID -> addStep(
platform.choose(
() ->
new CreateSnapshotPrimaryDataRowIdsStep(
bigQuerySnapshotPdao, snapshotDao, snapshotService, snapshotReq),
() ->
new CreateSnapshotByRowIdParquetFilesAzureStep(
azureSynapsePdao, snapshotService, snapshotReq, snapshotId)));
case BYROWID ->
addStep(
platform.choose(
() ->
new CreateSnapshotPrimaryDataRowIdsStep(
bigQuerySnapshotPdao, snapshotDao, snapshotService, snapshotReq),
() ->
new CreateSnapshotByRowIdParquetFilesAzureStep(
azureSynapsePdao, snapshotService, snapshotReq, snapshotId)));
case BYREQUESTID -> {
addStep(new CreateSnapshotSamGroupNameStep(snapshotId, iamService));
addStep(new CreateSnapshotSamGroupStep(iamService));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,12 +86,15 @@ static String generateSummaryForCriteriaGroup(
.map(
criteria ->
switch (criteria.getKind()) {
case LIST -> generateSummaryForCriteria(
(SnapshotBuilderProgramDataListCriteria) criteria, settings);
case RANGE -> generateSummaryForCriteria(
(SnapshotBuilderProgramDataRangeCriteria) criteria, settings);
case DOMAIN -> generateSummaryForCriteria(
(SnapshotBuilderDomainCriteria) criteria, settings);
case LIST ->
generateSummaryForCriteria(
(SnapshotBuilderProgramDataListCriteria) criteria, settings);
case RANGE ->
generateSummaryForCriteria(
(SnapshotBuilderProgramDataRangeCriteria) criteria, settings);
case DOMAIN ->
generateSummaryForCriteria(
(SnapshotBuilderDomainCriteria) criteria, settings);
})
.collect(Collectors.joining("\n")));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,11 +70,12 @@ public String renderSQL(SqlRenderContext context) {
return switch (dataType) {
case STRING -> stringVal == null ? "NULL" : "'" + sqlEscape(stringVal) + "'";
case INT64 -> String.valueOf(int64Val);
case BOOLEAN -> context
.getPlatform()
// In T-SQL, TRUE is not a keyword, so we need to use 1 instead. The JDBC API converts
// the 1 to a boolean TRUE when the query result is processed.
.choose(() -> String.valueOf(booleanVal), () -> booleanVal ? "1" : "0");
case BOOLEAN ->
context
.getPlatform()
// In T-SQL, TRUE is not a keyword, so we need to use 1 instead. The JDBC API converts
// the 1 to a boolean TRUE when the query result is processed.
.choose(() -> String.valueOf(booleanVal), () -> booleanVal ? "1" : "0");
case DATE -> "DATE('" + dateVal.toString() + "')";
case DOUBLE -> "FLOAT('" + doubleVal + "')";
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ public Query generateQuery(SnapshotBuilderDomainOption domainOption, int concept
.orderBy(List.of(new OrderByVariable(conceptName, OrderByDirection.ASCENDING)))
.build();
}

/**
* Filter concept to only allow standard concepts. See <a
* href="https://www.ohdsi.org/web/wiki/doku.php?id=documentation:vocabulary:standard_classification_and_source_concepts">Standard,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ public static int getTableTotalRowCount(FSContainerInterface tdrResource, String
return 0;
}
}

/*
* WARNING: Ensure input parameters are validated before executing this method!
*/
Expand Down
1 change: 1 addition & 0 deletions src/test/java/bio/terra/common/ParquetUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ public static List<Map<String, Object>> readGcsParquetRecords(
throw new RuntimeException("Could not write " + url + " to local temp file", e);
}
}

/**
* Given a signed Azure URL, this method will read through the parquet data specified as list of
* maps that are keyed on column name and whose values are string representations of the parquet
Expand Down
15 changes: 9 additions & 6 deletions src/test/java/bio/terra/integration/DataRepoFixtures.java
Original file line number Diff line number Diff line change
Expand Up @@ -567,8 +567,9 @@ public DataRepoResponse<Object> addPolicyMemberRaw(
case DATASET -> "/api/repository/v1/datasets/";
case DATASNAPSHOT -> "/api/repository/v1/snapshots/";
case SPEND_PROFILE -> "/api/resources/v1/profiles/";
default -> throw new IllegalArgumentException(
"Policy member addition undefined for IamResourceType " + iamResourceType);
default ->
throw new IllegalArgumentException(
"Policy member addition undefined for IamResourceType " + iamResourceType);
};
String path = pathPrefix + resourceId + "/policies/" + role.toString() + "/members";
return dataRepoClient.post(user, path, TestUtils.mapToJson(req), new TypeReference<>() {});
Expand Down Expand Up @@ -604,8 +605,9 @@ public DataRepoResponse<List<String>> retrieveUserRolesRaw(
switch (iamResourceType) {
case DATASET -> "/api/repository/v1/datasets/";
case DATASNAPSHOT -> "/api/repository/v1/snapshots/";
default -> throw new IllegalArgumentException(
"Role fetch undefined for IamResourceType " + iamResourceType);
default ->
throw new IllegalArgumentException(
"Role fetch undefined for IamResourceType " + iamResourceType);
};
String path = pathPrefix + resourceId + "/roles";

Expand All @@ -628,8 +630,9 @@ public DataRepoResponse<PolicyResponse> retrievePoliciesRaw(
case DATASET -> "/api/repository/v1/datasets/";
case DATASNAPSHOT -> "/api/repository/v1/snapshots/";
case SPEND_PROFILE -> "/api/resources/v1/profiles/";
default -> throw new IllegalArgumentException(
"Policy fetch undefined for IamResourceType " + iamResourceType);
default ->
throw new IllegalArgumentException(
"Policy fetch undefined for IamResourceType " + iamResourceType);
};
String path = pathPrefix + resourceId + "/policies";

Expand Down
Loading
Loading