Skip to content

Commit 146cd55

Browse files
medbmedb
authored
Run integration tests in presubmit (GoogleCloudDataproc#190)
1 parent f4d02a6 commit 146cd55

File tree

9 files changed

+85
-61
lines changed

9 files changed

+85
-61
lines changed

‎bigquery/src/test/java/com/google/cloud/hadoop/io/bigquery/AbstractBigQueryIoIntegrationTestBase.java

+27-24
Original file line numberDiff line numberDiff line change
@@ -128,37 +128,40 @@ protected abstract Map<String, Object> readRecord(RecordReader<?, T> recordReade
128128
* intended for BigQueryFactory and adding them as GCS-equivalent credential settings.
129129
*/
130130
public static Configuration getConfigForGcsFromBigquerySettings(
131-
String projectIdvalue, String testBucket) {
131+
String projectIdValue, String testBucket) {
132132
TestConfiguration configuration = TestConfiguration.getInstance();
133133

134-
String bigqueryServiceAccount = configuration.getServiceAccount();
135-
if (Strings.isNullOrEmpty(bigqueryServiceAccount)) {
136-
bigqueryServiceAccount = System.getenv(BigQueryFactory.BIGQUERY_SERVICE_ACCOUNT);
134+
String serviceAccount = configuration.getServiceAccount();
135+
if (Strings.isNullOrEmpty(serviceAccount)) {
136+
serviceAccount = System.getenv(BigQueryFactory.BIGQUERY_SERVICE_ACCOUNT);
137137
}
138138

139-
String bigqueryPrivateKeyFile = configuration.getPrivateKeyFile();
140-
if (Strings.isNullOrEmpty(bigqueryPrivateKeyFile)) {
141-
bigqueryPrivateKeyFile = System.getenv(BigQueryFactory.BIGQUERY_PRIVATE_KEY_FILE);
139+
String privateKeyFile = configuration.getPrivateKeyFile();
140+
if (Strings.isNullOrEmpty(privateKeyFile)) {
141+
privateKeyFile = System.getenv(BigQueryFactory.BIGQUERY_PRIVATE_KEY_FILE);
142142
}
143-
Configuration config = new Configuration();
144-
config.set(
145-
BIGQUERY_CONFIG_PREFIX + HadoopCredentialConfiguration.ENABLE_SERVICE_ACCOUNTS_SUFFIX,
146-
"true");
147-
config.set(
148-
BIGQUERY_CONFIG_PREFIX + HadoopCredentialConfiguration.SERVICE_ACCOUNT_EMAIL_SUFFIX,
149-
bigqueryServiceAccount);
150-
config.set(
151-
BIGQUERY_CONFIG_PREFIX + HadoopCredentialConfiguration.SERVICE_ACCOUNT_KEYFILE_SUFFIX,
152-
bigqueryPrivateKeyFile);
153-
config.set(
154-
GoogleHadoopFileSystemConfiguration.AUTH_SERVICE_ACCOUNT_KEY_FILE.getKey(),
155-
bigqueryPrivateKeyFile);
156-
config.set(
157-
GoogleHadoopFileSystemConfiguration.AUTH_SERVICE_ACCOUNT_EMAIL.getKey(),
158-
bigqueryServiceAccount);
159-
config.set(GoogleHadoopFileSystemConfiguration.GCS_PROJECT_ID.getKey(), projectIdvalue);
160143

144+
Configuration config = new Configuration();
161145
config.set("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem");
146+
config.set(GoogleHadoopFileSystemConfiguration.GCS_PROJECT_ID.getKey(), projectIdValue);
147+
148+
if (serviceAccount != null && privateKeyFile != null) {
149+
config.setBoolean(
150+
BIGQUERY_CONFIG_PREFIX + HadoopCredentialConfiguration.ENABLE_SERVICE_ACCOUNTS_SUFFIX,
151+
true);
152+
config.set(
153+
BIGQUERY_CONFIG_PREFIX + HadoopCredentialConfiguration.SERVICE_ACCOUNT_EMAIL_SUFFIX,
154+
serviceAccount);
155+
config.set(
156+
BIGQUERY_CONFIG_PREFIX + HadoopCredentialConfiguration.SERVICE_ACCOUNT_KEYFILE_SUFFIX,
157+
privateKeyFile);
158+
config.set(
159+
GoogleHadoopFileSystemConfiguration.AUTH_SERVICE_ACCOUNT_EMAIL.getKey(), serviceAccount);
160+
config.set(
161+
GoogleHadoopFileSystemConfiguration.AUTH_SERVICE_ACCOUNT_KEY_FILE.getKey(),
162+
privateKeyFile);
163+
}
164+
162165
return config;
163166
}
164167

‎cloudbuild/cloudbuild.yaml

+36-4
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ steps:
99
id: 'unit-tests-hadoop2'
1010
waitFor: ['docker-build']
1111
entrypoint: 'bash'
12-
args: ['/bigdata-interop/cloudbuild/presubmit.sh', 'hadoop2']
12+
args: ['/bigdata-interop/cloudbuild/presubmit.sh', 'hadoop2', 'unittest']
1313
env:
1414
- 'CODECOV_TOKEN=$_CODECOV_TOKEN'
1515
- 'VCS_BRANCH_NAME=$BRANCH_NAME'
@@ -22,13 +22,45 @@ steps:
2222
id: 'unit-tests-hadoop3'
2323
waitFor: ['docker-build']
2424
entrypoint: 'bash'
25-
args: ['/bigdata-interop/cloudbuild/presubmit.sh', 'hadoop3']
25+
args: ['/bigdata-interop/cloudbuild/presubmit.sh', 'hadoop3', 'unittest']
2626
env:
2727
- 'CODECOV_TOKEN=$_CODECOV_TOKEN'
2828
- 'VCS_BRANCH_NAME=$BRANCH_NAME'
2929
- 'VCS_COMMIT_ID=$COMMIT_SHA'
3030
- 'VCS_TAG=$TAG_NAME'
3131
- 'CI_BUILD_ID=$BUILD_ID'
32-
32+
33+
# 4. Run Hadoop 2 integration tests concurrently with Hadoop 2 and Hadoop 3 unit tests
34+
- name: 'gcr.io/$PROJECT_ID/dataproc-bigdata-interop-presubmit'
35+
id: 'integration-tests-hadoop2'
36+
waitFor: ['docker-build']
37+
entrypoint: 'bash'
38+
args: ['/bigdata-interop/cloudbuild/presubmit.sh', 'hadoop2', 'integrationtest']
39+
env:
40+
- 'GCS_TEST_PROJECT_ID=$PROJECT_ID'
41+
- 'CODECOV_TOKEN=$_CODECOV_TOKEN'
42+
- 'VCS_BRANCH_NAME=$BRANCH_NAME'
43+
- 'VCS_COMMIT_ID=$COMMIT_SHA'
44+
- 'VCS_TAG=$TAG_NAME'
45+
- 'CI_BUILD_ID=$BUILD_ID'
46+
47+
# 5. Run Hadoop 3 integration tests sequentially after `integration-tests-hadoop2` step
48+
- name: 'gcr.io/$PROJECT_ID/dataproc-bigdata-interop-presubmit'
49+
id: 'integration-tests-hadoop3'
50+
waitFor: ['integration-tests-hadoop2']
51+
entrypoint: 'bash'
52+
args: ['/bigdata-interop/cloudbuild/presubmit.sh', 'hadoop3', 'integrationtest']
53+
env:
54+
- 'GCS_TEST_PROJECT_ID=$PROJECT_ID'
55+
- 'CODECOV_TOKEN=$_CODECOV_TOKEN'
56+
- 'VCS_BRANCH_NAME=$BRANCH_NAME'
57+
- 'VCS_COMMIT_ID=$COMMIT_SHA'
58+
- 'VCS_TAG=$TAG_NAME'
59+
- 'CI_BUILD_ID=$BUILD_ID'
60+
61+
# Tests take on average 17 minutes to run
62+
timeout: 1800s
63+
3364
options:
34-
machineType: 'N1_HIGHCPU_8'
65+
machineType: 'N1_HIGHCPU_32'
66+

‎cloudbuild/presubmit.sh

+8-6
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,16 @@
1717
set -euxo pipefail
1818

1919
readonly HADOOP_PROFILE="$1"
20+
readonly TEST_TYPE="${2:-unittest}"
2021

2122
cd /bigdata-interop
2223

23-
# Print Maven info
24-
./mvnw -v
25-
26-
# Run unit tests and generate test coverage report
27-
./mvnw -B -e "-P${HADOOP_PROFILE}" -Pcoverage clean verify
24+
# Run unit or integration tests and generate test coverage report
25+
if [[ $TEST_TYPE == unittest ]]; then
26+
./mvnw -B -e "-P${HADOOP_PROFILE}" -Pcoverage clean verify
27+
else
28+
./mvnw -B -e "-P${HADOOP_PROFILE}" -Pintegration-test -Pcoverage clean verify
29+
fi
2830

2931
# Upload test coverage report to Codecov
30-
bash <(curl -s https://codecov.io/bash) -K -F "${HADOOP_PROFILE}unittest"
32+
bash <(curl -s https://codecov.io/bash) -K -F "${HADOOP_PROFILE}${TEST_TYPE}"

‎coverage/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,4 +66,4 @@
6666
<version>${hadoop.identifier}-${bigdataoss.version}</version>
6767
</dependency>
6868
</dependencies>
69-
</project>
69+
</project>

‎gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java

+2-8
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,7 @@
1414

1515
package com.google.cloud.hadoop.fs.gcs;
1616

17-
import static com.google.cloud.hadoop.gcsio.testing.TestConfiguration.GCS_TEST_PRIVATE_KEYFILE;
1817
import static com.google.cloud.hadoop.gcsio.testing.TestConfiguration.GCS_TEST_PROJECT_ID;
19-
import static com.google.cloud.hadoop.gcsio.testing.TestConfiguration.GCS_TEST_SERVICE_ACCOUNT;
2018
import static com.google.common.base.Preconditions.checkNotNull;
2119

2220
import com.google.cloud.hadoop.gcsio.testing.TestConfiguration;
@@ -44,12 +42,8 @@ public static Configuration getTestConfig() {
4442
TestConfiguration testConfiguration = TestConfiguration.getInstance();
4543
String projectId =
4644
checkNotNull(testConfiguration.getProjectId(), ENV_VAR_MSG_FMT, GCS_TEST_PROJECT_ID);
47-
String privateKeyFile =
48-
checkNotNull(
49-
testConfiguration.getPrivateKeyFile(), ENV_VAR_MSG_FMT, GCS_TEST_PRIVATE_KEYFILE);
50-
String serviceAccount =
51-
checkNotNull(
52-
testConfiguration.getServiceAccount(), ENV_VAR_MSG_FMT, GCS_TEST_SERVICE_ACCOUNT);
45+
String privateKeyFile = testConfiguration.getPrivateKeyFile();
46+
String serviceAccount = testConfiguration.getPrivateKeyFile();
5347

5448
Configuration config = new Configuration();
5549
config.set(GoogleHadoopFileSystemConfiguration.GCS_PROJECT_ID.getKey(), projectId);

‎gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemTestBase.java

-6
Original file line numberDiff line numberDiff line change
@@ -76,12 +76,6 @@ protected static Configuration loadConfig(
7676
assertWithMessage("Expected value for env var %s", TestConfiguration.GCS_TEST_PROJECT_ID)
7777
.that(projectId)
7878
.isNotNull();
79-
assertWithMessage("Expected value for env var %s", TestConfiguration.GCS_TEST_SERVICE_ACCOUNT)
80-
.that(serviceAccount)
81-
.isNotNull();
82-
assertWithMessage("Expected value for env var %s", TestConfiguration.GCS_TEST_PRIVATE_KEYFILE)
83-
.that(privateKeyFile)
84-
.isNotNull();
8579
Configuration config = new Configuration();
8680
config.set(GoogleHadoopFileSystemConfiguration.GCS_PROJECT_ID.getKey(), projectId);
8781
config.set(

‎gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/GoogleContract.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,10 @@ public GoogleContract(Configuration conf, TestBucketHelper bucketHelper) {
3838

3939
TestConfiguration testConf = TestConfiguration.getInstance();
4040
if (testConf.getProjectId() != null) {
41-
conf.setBoolean(AUTH_SERVICE_ACCOUNT_ENABLE.getKey(), true);
4241
conf.set(GCS_PROJECT_ID.getKey(), testConf.getProjectId());
42+
}
43+
if (testConf.getServiceAccount() != null && testConf.getPrivateKeyFile() != null) {
44+
conf.setBoolean(AUTH_SERVICE_ACCOUNT_ENABLE.getKey(), true);
4345
conf.set(AUTH_SERVICE_ACCOUNT_EMAIL.getKey(), testConf.getServiceAccount());
4446
conf.set(AUTH_SERVICE_ACCOUNT_KEY_FILE.getKey(), testConf.getPrivateKeyFile());
4547
}

‎gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java

+4-8
Original file line numberDiff line numberDiff line change
@@ -14,19 +14,13 @@
1414

1515
package com.google.cloud.hadoop.gcsio.testing;
1616

17-
18-
19-
/**
20-
* Access to test configurations values.
21-
*/
17+
/** Access to test configurations values. */
2218
public abstract class TestConfiguration {
2319
public static final String GCS_TEST_PROJECT_ID = "GCS_TEST_PROJECT_ID";
2420
public static final String GCS_TEST_SERVICE_ACCOUNT = "GCS_TEST_SERVICE_ACCOUNT";
2521
public static final String GCS_TEST_PRIVATE_KEYFILE = "GCS_TEST_PRIVATE_KEYFILE";
2622

27-
/**
28-
* Environment-based test configuration.
29-
*/
23+
/** Environment-based test configuration. */
3024
public static class EnvironmentBasedTestConfiguration extends TestConfiguration {
3125
@Override
3226
public String getProjectId() {
@@ -53,6 +47,8 @@ private static class LazyHolder {
5347
}
5448

5549
public abstract String getProjectId();
50+
5651
public abstract String getServiceAccount();
52+
5753
public abstract String getPrivateKeyFile();
5854
}

‎gcsio/src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,11 @@ public static Credential getCredential() throws IOException {
6060
String serviceAccount = TestConfiguration.getInstance().getServiceAccount();
6161
String privateKeyfile = TestConfiguration.getInstance().getPrivateKeyFile();
6262

63-
assertWithMessage("privateKeyfile must not be null").that(privateKeyfile).isNotNull();
64-
assertWithMessage("serviceAccount must not be null").that(serviceAccount).isNotNull();
63+
CredentialFactory credentialFactory = new CredentialFactory();
6564
try {
66-
CredentialFactory credentialFactory = new CredentialFactory();
65+
if (serviceAccount == null || privateKeyfile == null) {
66+
return credentialFactory.getCredentialFromMetadataServiceAccount();
67+
}
6768
return credentialFactory.getCredentialFromPrivateKeyServiceAccount(
6869
serviceAccount,
6970
privateKeyfile,

0 commit comments

Comments
 (0)