Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
dda60be
[DCJ-400] Upgrade Spring Boot 3.2.4 -> 3.3.2
okotsopoulos Aug 7, 2024
b6cfa32
Upgrade com.google.cloud:libraries-bom 26.30.0 -> 26.43.0
okotsopoulos Aug 7, 2024
e43eea2
Explicitly pull in org.apache.commons:commons-compress for job, srccl…
okotsopoulos Aug 7, 2024
559b311
Upgrade com.google.cloud.tools.jib to latest 3.4.3
okotsopoulos Aug 7, 2024
4f6b9f2
Upgrade plugin org.gradle.test-retry 1.5.8 -> 1.5.10
okotsopoulos Aug 7, 2024
eaeea23
Upgrade plugin com.gorylenko.gradle-git-properties 2.4.1 -> 2.4.2
okotsopoulos Aug 7, 2024
5a33c6e
Update antlr dependencies
okotsopoulos Aug 7, 2024
d9d5642
Update Shedlock dependencies
okotsopoulos Aug 7, 2024
d404e8f
Upgrade bio.terra:terra-policy-client 1.0.11-SNAPSHOT -> 1.0.15-SNAPSHOT
okotsopoulos Aug 7, 2024
ad13aa2
Upgrade bio.terra:externalcreds-client-resttemplate 1.3.0-SNAPSHOT ->…
okotsopoulos Aug 7, 2024
be73823
Merge branch 'develop' into okotsopo-DCJ-400-minor-patch-deps
okotsopoulos Aug 7, 2024
60f4f37
Upgrade Azure dependencies
okotsopoulos Aug 7, 2024
5cfc651
Upgrade liquibase-core to latest
okotsopoulos Aug 7, 2024
a0a79fb
Driveby - pull in Sentry dependency compatible with Spring Boot 3.x
okotsopoulos Aug 7, 2024
09bfbfe
Upgrade opentelemetry-instrumentation-annotations 2.2.0 -> 2.6.0
okotsopoulos Aug 7, 2024
f6e61e9
Upgrade org.apache.parquet deps 1.12.0 -> 1.14.1
okotsopoulos Aug 7, 2024
4501079
Upgrade org.apache.hadoop:hadoop-common 3.3.1 -> 3.4.0
okotsopoulos Aug 7, 2024
53c9641
Upgrade au.com.dius.pact.provider deps 4.3.19 -> 4.6.12
okotsopoulos Aug 7, 2024
8f5d0bf
Remove groovy runtime dependency
okotsopoulos Aug 7, 2024
bd13d94
Upgrade Pico CLI, Zonky DB test dependencies
okotsopoulos Aug 7, 2024
449a638
Fix Dependabot config's exclusion of Spotless from grouped updates
okotsopoulos Aug 7, 2024
c8b79a1
org.apache.hadoop:hadoop-mapreduce-client-core needed for integration…
okotsopoulos Aug 8, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ updates:
patterns:
- "*"
exclude-patterns:
- "spotless-plugin-gradle" # likely to require reformatting of code
- "com.diffplug.spotless" # likely to require reformatting of code
update-types:
- "minor"
- "patch"
Expand Down
94 changes: 41 additions & 53 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -22,25 +22,25 @@ buildscript {
dependencies {
classpath('io.swagger.codegen.v3:swagger-codegen:3.0.52')
// Required for gradle liquibase plugin
classpath ('org.liquibase:liquibase-core:4.26.0')
classpath ('org.liquibase:liquibase-core:4.29.1')
}
}

plugins {
id 'com.google.cloud.tools.jib' version '3.2.0'
id 'com.google.cloud.tools.jib' version '3.4.3'
id 'org.liquibase.gradle' version '3.0.0'
id 'org.gradle.test-retry' version '1.5.8'
id 'org.gradle.test-retry' version '1.5.10'
id 'antlr'
id 'org.hidetake.swagger.generator' version '2.19.2'
id 'org.springframework.boot' version '3.2.4'
id 'org.springframework.boot' version '3.3.2'
id 'idea'
id 'java'
id 'io.spring.dependency-management' version '1.1.4'
id 'io.spring.dependency-management' version '1.1.6'
id 'jacoco'
id 'com.diffplug.spotless' version '6.7.1'
id 'com.dorongold.task-tree' version '4.0.0'
// enables release info in sentry events
id 'com.gorylenko.gradle-git-properties' version '2.4.1'
id 'com.gorylenko.gradle-git-properties' version '2.4.2'
id 'org.sonarqube' version '4.2.1.3168'
id 'com.srcclr.gradle' version '3.1.12'
}
Expand Down Expand Up @@ -162,19 +162,13 @@ configurations {
runtimeClasspath
}

// Spring Boot 3.2.4 pulls in opentelemetry-bom 1.31.0.
// It must have version >= 1.34.1 for compatibility with terra-common-lib 1.1.10:
ext['opentelemetry.version'] = '1.36.0'
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now 1.37.0.

// Spring Boot 3.2.4 pulls in io.netty:netty-bom 4.1.107.Final which is impacted by CVE-2024-29025.
ext['netty.version'] = '4.1.108.Final'
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.


dependencies {
implementation 'com.google.apis:google-api-services-serviceusage:v1-rev20230215-2.0.0'
implementation 'com.google.apis:google-api-services-appengine:v1-rev20230206-2.0.0'
implementation 'com.google.apis:google-api-services-oauth2:v2-rev20200213-2.0.0'
implementation 'com.google.apis:google-api-services-iam:v1-rev20230209-2.0.0'

implementation platform('com.google.cloud:libraries-bom:26.30.0')
implementation platform('com.google.cloud:libraries-bom:26.43.0')
implementation 'com.google.cloud:google-cloud-billing'
implementation 'com.google.cloud:google-cloud-resourcemanager'
implementation 'com.google.cloud:google-cloud-bigquery'
Expand All @@ -184,13 +178,15 @@ dependencies {
implementation 'com.google.cloud:spring-cloud-gcp-starter-logging:4.9.0'
implementation 'com.google.http-client:google-http-client'

implementation 'org.apache.commons:commons-compress:1.26.2' // For srcclr, jib plugin conflict
// More info: https://discuss.gradle.org/t/plugin-dependency-conflict-with-jib-and-srcclr/42355
implementation 'org.apache.commons:commons-dbcp2' // For database connection support
implementation 'org.apache.commons:commons-lang3'
implementation 'org.apache.commons:commons-collections4'
implementation 'org.apache.directory.studio:org.apache.commons.io:2.4'
implementation 'org.apache.httpcomponents.client5:httpclient5'

implementation 'org.liquibase:liquibase-core:4.26.0'
implementation 'org.liquibase:liquibase-core'

implementation 'org.codehaus.janino:janino' // Provides if/else xml parsing for logback config
implementation 'org.springframework.boot:spring-boot-starter-web'
Expand All @@ -201,90 +197,82 @@ dependencies {
swaggerCodegen 'io.swagger.codegen.v3:swagger-codegen-cli'

implementation 'org.springframework:spring-jdbc'
implementation 'org.antlr:ST4:4.3' // String templating
implementation 'org.antlr:ST4:4.3.4' // String templating

implementation 'org.springframework.boot:spring-boot-starter-thymeleaf'
implementation 'jakarta.servlet.jsp.jstl:jakarta.servlet.jsp.jstl-api'

implementation 'com.microsoft.sqlserver:mssql-jdbc:11.2.3.jre17'

// For distributed locking of Spring @Scheduled tasks across multiple instances
implementation 'net.javacrumbs.shedlock:shedlock-provider-jdbc-template:5.2.0'
implementation 'net.javacrumbs.shedlock:shedlock-spring:5.2.0'
implementation 'net.javacrumbs.shedlock:shedlock-provider-jdbc-template:5.14.0'
implementation 'net.javacrumbs.shedlock:shedlock-spring:5.14.0'

implementation 'bio.terra:terra-common-lib:1.1.17-SNAPSHOT'
implementation 'org.broadinstitute.dsde.workbench:sam-client_2.13:v0.0.241'
implementation 'bio.terra:terra-policy-client:1.0.11-SNAPSHOT'
implementation 'bio.terra:terra-policy-client:1.0.15-SNAPSHOT'
implementation 'bio.terra:terra-resource-buffer-client:0.198.42-SNAPSHOT'
implementation 'bio.terra:externalcreds-client-resttemplate:1.3.0-SNAPSHOT'
implementation 'bio.terra:externalcreds-client-resttemplate:1.45.0-SNAPSHOT'

implementation 'org.glassfish.jersey.inject:jersey-hk2'

implementation 'com.squareup.okhttp3:okhttp'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'io.micrometer:micrometer-registry-prometheus'

implementation 'com.fasterxml.jackson.core:jackson-core:2.15.3'
implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.3'
implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.3'
implementation 'com.fasterxml.jackson.core:jackson-core'
implementation 'com.fasterxml.jackson.core:jackson-annotations'
implementation 'com.fasterxml.jackson.core:jackson-databind'

// Azure related dependencies
implementation 'com.azure:azure-identity:1.11.1'
implementation 'com.azure.resourcemanager:azure-resourcemanager:2.34.0'
implementation 'com.azure:azure-identity:1.13.2'
implementation 'com.azure.resourcemanager:azure-resourcemanager:2.41.0'
implementation 'com.azure.resourcemanager:azure-resourcemanager-loganalytics:1.0.0'
implementation 'com.azure.resourcemanager:azure-resourcemanager-securityinsights:1.0.0-beta.4'
implementation 'com.azure:azure-storage-common:12.24.1'
implementation 'com.azure:azure-storage-file-datalake:12.18.1'
implementation 'com.azure:azure-data-tables:12.3.18'
implementation 'com.azure:azure-storage-common:12.26.0'
implementation 'com.azure:azure-storage-file-datalake:12.20.0'
implementation 'com.azure:azure-data-tables:12.4.3'

implementation platform('io.sentry:sentry-bom:7.13.0') //import bom
implementation('io.sentry:sentry-spring-boot-starter')
implementation('io.sentry:sentry-spring-boot-starter-jakarta')
implementation('io.sentry:sentry-logback')

// OpenTelemetry @WithSpan annotations:
implementation 'io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations:2.2.0'

testImplementation 'org.apache.parquet:parquet-common:1.12.0'
testImplementation 'org.apache.parquet:parquet-hadoop:1.12.0'
testImplementation 'org.apache.parquet:parquet-hadoop-bundle:1.12.0'
testImplementation 'org.apache.parquet:parquet-encoding:1.12.0'
testImplementation 'org.apache.parquet:parquet-column:1.12.0'
testImplementation ('org.apache.hadoop:hadoop-common:3.3.1') {
implementation 'io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations:2.6.0'

testImplementation 'org.apache.parquet:parquet-common:1.14.1'
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For repeated values like this we could use a variable

dependencies {
  ext {
    parquet = '1.14.1'
  }
  testImplementation "org.apache.parquet:parquet-common:${parquet}"
}

testImplementation 'org.apache.parquet:parquet-hadoop:1.14.1'
testImplementation 'org.apache.parquet:parquet-hadoop-bundle:1.14.1'
testImplementation 'org.apache.parquet:parquet-encoding:1.14.1'
testImplementation 'org.apache.parquet:parquet-column:1.14.1'
testImplementation ('org.apache.hadoop:hadoop-common:3.4.0') {
exclude group: 'com.sun.jersey', module: 'jersey-core'
exclude group: 'com.sun.jersey', module: 'jersey-servlet'
exclude group: 'com.sun.jersey', module: 'jersey-json'
exclude group: 'com.sun.jersey', module: 'jersey-server'
exclude group: 'org.slf4j', module: 'slf4j-reload4j'
}
testImplementation ('org.apache.hadoop:hadoop-azure:3.3.1') {
testImplementation('org.apache.hadoop:hadoop-mapreduce-client-core:3.4.0') {
exclude group: 'com.sun.jersey', module: 'jersey-core'
exclude group: 'com.sun.jersey', module: 'jersey-servlet'
exclude group: 'com.sun.jersey', module: 'jersey-json'
exclude group: 'com.sun.jersey', module: 'jersey-server'
exclude group: 'org.slf4j', module: 'slf4j-reload4j'
}
testImplementation('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.1') {
exclude group: 'com.sun.jersey', module: 'jersey-core'
exclude group: 'com.sun.jersey', module: 'jersey-servlet'
exclude group: 'com.sun.jersey', module: 'jersey-json'
exclude group: 'com.sun.jersey', module: 'jersey-server'
}

testImplementation 'au.com.dius.pact.provider:junit5:4.3.19'
testImplementation 'au.com.dius.pact.provider:junit5spring:4.3.19'

antlr 'org.antlr:antlr4:4.8'
testImplementation 'au.com.dius.pact.provider:junit5:4.6.12'
testImplementation 'au.com.dius.pact.provider:junit5spring:4.6.12'

// Need groovy on the class path for the logback config. Could use XML and skip this dependency,
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use XML now, and logback hasn't supported groovy since 2021: https://logback.qos.ch/news.html#1.2.9

// but the groovy config is... well... groovy.
runtimeOnly 'org.codehaus.groovy:groovy:3.0.7'
antlr 'org.antlr:antlr4:4.13.2'

liquibaseRuntime 'org.liquibase:liquibase-core'
liquibaseRuntime 'org.postgresql:postgresql'
liquibaseRuntime 'info.picocli:picocli:4.7.5'
liquibaseRuntime 'info.picocli:picocli:4.7.6'

testImplementation 'org.junit.vintage:junit-vintage-engine'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
testImplementation 'io.zonky.test:embedded-database-spring-test:2.5.0'
testImplementation 'io.zonky.test:embedded-postgres:2.0.6'
testImplementation 'io.zonky.test:embedded-database-spring-test:2.5.1'
testImplementation 'io.zonky.test:embedded-postgres:2.0.7'
implementation enforcedPlatform('io.zonky.test.postgres:embedded-postgres-binaries-bom:12.8.0')

generatedCompile 'org.springframework.boot:spring-boot-starter-web'
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/bio/terra/service/auth/ras/EcmService.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import bio.terra.common.iam.AuthenticatedUserRequest;
import bio.terra.externalcreds.api.PassportApi;
import bio.terra.externalcreds.client.ApiClient;
import bio.terra.externalcreds.model.PassportProvider;
import bio.terra.externalcreds.model.RASv1Dot1VisaCriterion;
import bio.terra.externalcreds.model.ValidatePassportRequest;
import bio.terra.externalcreds.model.ValidatePassportResult;
Expand Down Expand Up @@ -37,7 +38,6 @@ public class EcmService {
private final ObjectMapper objectMapper;
private final OidcApiService oidcApiService;

private static final String RAS_PROVIDER = "ras";
@VisibleForTesting public static final String GA4GH_PASSPORT_V1_CLAIM = "ga4gh_passport_v1";
private static final String RAS_DBGAP_PERMISSIONS_CLAIM = "ras_dbgap_permissions";
private static final String RAS_CRITERIA_TYPE = "RASv1Dot1VisaCriterion";
Expand Down Expand Up @@ -98,7 +98,7 @@ public ValidatePassportResult validatePassport(ValidatePassportRequest validateP
*/
public String getRasProviderPassport(AuthenticatedUserRequest userReq) {
try {
return oidcApiService.getOidcApi(userReq).getProviderPassport(RAS_PROVIDER);
return oidcApiService.getOidcApi(userReq).getProviderPassport(PassportProvider.RAS);
} catch (HttpClientErrorException ex) {
if (ex.getStatusCode() == HttpStatus.NOT_FOUND) {
return null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ public static TpsPolicyInput getProtectedDataPolicyInput() {
public TpsPaoGetResult getPao(UUID resourceId) {
TpsApi tpsApi = policyApiService.getPolicyApi();
try {
return tpsApi.getPao(resourceId);
return tpsApi.getPao(resourceId, false);
} catch (ApiException e) {
throw convertApiException(e);
}
Expand Down
7 changes: 5 additions & 2 deletions src/test/java/bio/terra/common/BQTestUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,11 @@ public static void mockBQQuery(

private static Answer<TableResult> mockAnswer(Schema schema, List<Map<String, String>> results) {
return a ->
new TableResult(
schema, results.size(), new PageImpl<>(null, null, convertValues(results, schema)));
TableResult.newBuilder()
.setSchema(schema)
.setTotalRows((long) results.size())
.setPageNoSchema(new PageImpl<>(null, null, convertValues(results, schema)))
.build();
}

private static List<FieldValueList> convertValues(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

Expand All @@ -13,6 +12,7 @@
import bio.terra.common.fixtures.AuthenticationFixtures;
import bio.terra.common.iam.AuthenticatedUserRequest;
import bio.terra.externalcreds.api.OidcApi;
import bio.terra.externalcreds.model.PassportProvider;
import bio.terra.service.auth.ras.EcmService;
import bio.terra.service.auth.ras.OidcApiService;
import bio.terra.service.auth.ras.RasDbgapPermissions;
Expand Down Expand Up @@ -56,7 +56,7 @@ void testGetRasProviderPassport() {
String passport = "passportJwt";
HttpClientErrorException shouldCatch = new HttpClientErrorException(HttpStatus.NOT_FOUND);
HttpClientErrorException shouldThrow = new HttpClientErrorException(HttpStatus.UNAUTHORIZED);
when(oidcApi.getProviderPassport(any()))
when(oidcApi.getProviderPassport(PassportProvider.RAS))
.thenReturn(passport)
.thenThrow(shouldCatch)
.thenThrow(shouldThrow);
Expand All @@ -75,7 +75,7 @@ void testGetRasProviderPassport() {

@Test
void testGetRasDbgapPermissionsNoPassport() throws Exception {
when(oidcApi.getProviderPassport(any()))
when(oidcApi.getProviderPassport(PassportProvider.RAS))
.thenThrow(new HttpClientErrorException(HttpStatus.NOT_FOUND));
assertThat(
"No RAS dbGaP permissions when a user doesn't have a passport",
Expand All @@ -85,7 +85,7 @@ void testGetRasDbgapPermissionsNoPassport() throws Exception {

@Test
void testGetRasDbgapPermissionsNoVisaClaim() throws Exception {
when(oidcApi.getProviderPassport(any())).thenReturn(toJwtToken("{}"));
when(oidcApi.getProviderPassport(PassportProvider.RAS)).thenReturn(toJwtToken("{}"));
assertThat(
"No RAS dbGaP permissions when a user's passport has no visa claim'",
ecmService.getRasDbgapPermissions(TEST_USER),
Expand All @@ -94,7 +94,7 @@ void testGetRasDbgapPermissionsNoVisaClaim() throws Exception {

@Test
void testGetRasDbgapPermissionsNoVisas() throws Exception {
when(oidcApi.getProviderPassport(any())).thenReturn(toPassportJwt(null));
when(oidcApi.getProviderPassport(PassportProvider.RAS)).thenReturn(toPassportJwt(null));
assertThat(
"No RAS dbGaP permissions when a user's passport has no visas",
ecmService.getRasDbgapPermissions(TEST_USER),
Expand All @@ -110,7 +110,7 @@ void testGetRasDbgapPermissionsInvalidVisas() throws Exception {
"ras_dbgap_permissions": "should throw InvalidDefinitionException"
}
""";
when(oidcApi.getProviderPassport(any())).thenReturn(toPassportJwt(invalidVisa));
when(oidcApi.getProviderPassport(PassportProvider.RAS)).thenReturn(toPassportJwt(invalidVisa));

assertThat(
"No RAS dbGaP permissions when a user's passport has invalid visas",
Expand Down Expand Up @@ -143,7 +143,7 @@ void testGetRasDbgapPermissionsValidVisas() throws Exception {
]
}
""";
when(oidcApi.getProviderPassport(any())).thenReturn(toPassportJwt(validVisa));
when(oidcApi.getProviderPassport(PassportProvider.RAS)).thenReturn(toPassportJwt(validVisa));

assertThat(
"Passport visa permissions are successfully decoded and unknown properties ignored",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
Expand All @@ -19,6 +20,7 @@
import bio.terra.policy.model.TpsComponent;
import bio.terra.policy.model.TpsObjectType;
import bio.terra.policy.model.TpsPaoCreateRequest;
import bio.terra.policy.model.TpsPaoGetResult;
import bio.terra.policy.model.TpsPaoUpdateRequest;
import bio.terra.policy.model.TpsPolicyInput;
import bio.terra.policy.model.TpsPolicyInputs;
Expand Down Expand Up @@ -209,7 +211,8 @@ void testStatusNotOk() throws Exception {
@Test
void getPao() throws ApiException {
mockPolicyApi();
policyService.getPao(snapshotId);
verify(tpsApi).getPao(snapshotId);
var expected = new TpsPaoGetResult().objectId(snapshotId);
when(tpsApi.getPao(snapshotId, false)).thenReturn(expected);
assertThat(policyService.getPao(snapshotId), equalTo(expected));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -1061,7 +1061,8 @@ void aggregateSnapshotTableTest() {

Page<FieldValueList> page = mockPage(listOfFieldValueList);

TableResult table = new TableResult(schema, 10, page);
TableResult table =
TableResult.newBuilder().setSchema(schema).setTotalRows(10L).setPageNoSchema(page).build();

List<BigQueryDataResultModel> result = BigQueryPdao.aggregateTableData(table);

Expand Down Expand Up @@ -1091,7 +1092,8 @@ void testAggregateTextColumnStats() {
"2"))))); // For some reason, it wants the numeric value passed in as a
// string

TableResult table = new TableResult(schema, 10, page);
TableResult table =
TableResult.newBuilder().setSchema(schema).setTotalRows(10L).setPageNoSchema(page).build();

List<ColumnStatisticsTextValue> result =
BigQueryPdao.aggregateTextColumnStats(table, columnName);
Expand Down
Loading