Skip to content

Commit 3494524

Browse files
davidh44davidh44
authored
Migration tool ObjectMetadata transforms (#5970)
* Migration tool ObjectMetadata transforms * Add transform for httpExpiresDate * Add null checks and use fromContentProvider if content length not set * Address comments
1 parent 3444966 commit 3494524

File tree

6 files changed

+564
-81
lines changed

6 files changed

+564
-81
lines changed

‎test/v2-migration-tests/src/test/resources/software/amazon/awssdk/v2migrationtests/maven-nocompile/after/src/main/java/foo/bar/S3Transforms.java

+39-4
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,53 @@
2323
import software.amazon.awssdk.core.async.AsyncRequestBody;
2424
import software.amazon.awssdk.services.s3.S3Client;
2525
import software.amazon.awssdk.services.s3.model.GeneratePresignedUrlRequest;
26+
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
2627
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
2728
import software.amazon.awssdk.transfer.s3.S3TransferManager;
2829
import software.amazon.awssdk.transfer.s3.model.UploadRequest;
2930

3031
public class S3Transforms {
3132

32-
void upload(S3TransferManager tm, String bucket, String key) {
33+
void upload_streamWithLiteralLength(S3TransferManager tm, String bucket, String key) {
34+
HeadObjectResponse metadata = HeadObjectResponse.builder()
35+
.build();
3336
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
34-
PutObjectRequest requestWithInputStream = PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location")
37+
PutObjectRequest requestWithStreamAndLiteralLength = PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location").contentLength(333L)
3538
.build();
36-
/*AWS SDK for Java v2 migration: When using InputStream to upload with TransferManager, you must specify Content-Length and ExecutorService.*/tm.upload(UploadRequest.builder().putObjectRequest(requestWithInputStream).requestBody(AsyncRequestBody.fromInputStream(inputStream, -1L, newExecutorServiceVariableToDefine)).build());
39+
/*AWS SDK for Java v2 migration: When using InputStream to upload with TransferManager, you must specify Content-Length and ExecutorService.*/tm.upload(UploadRequest.builder().putObjectRequest(requestWithStreamAndLiteralLength).requestBody(AsyncRequestBody.fromInputStream(inputStream, 333, newExecutorServiceVariableToDefine)).build());
40+
}
41+
42+
void upload_streamWithAssignedLength(S3TransferManager tm, String bucket, String key) {
43+
HeadObjectResponse metadata = HeadObjectResponse.builder()
44+
.build();
45+
long contentLen = 777;
46+
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
47+
PutObjectRequest requestWithStreamAndAssignedLength = PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location").contentLength(contentLen)
48+
.build();
49+
/*AWS SDK for Java v2 migration: When using InputStream to upload with TransferManager, you must specify Content-Length and ExecutorService.*/tm.upload(UploadRequest.builder().putObjectRequest(requestWithStreamAndAssignedLength).requestBody(AsyncRequestBody.fromInputStream(inputStream, contentLen, newExecutorServiceVariableToDefine)).build());
50+
}
51+
52+
void upload_streamWithoutLength(S3TransferManager tm, String bucket, String key) {
53+
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
54+
PutObjectRequest requestWithStreamAndNoLength = PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location")
55+
.build();
56+
/*AWS SDK for Java v2 migration: When using InputStream to upload with TransferManager, you must specify Content-Length and ExecutorService.*/tm.upload(UploadRequest.builder().putObjectRequest(requestWithStreamAndNoLength).requestBody(AsyncRequestBody.fromInputStream(inputStream, -1L, newExecutorServiceVariableToDefine)).build());
57+
}
58+
59+
void objectmetadata_unsupportedSetters(Date dateVal) {
60+
HeadObjectResponse metadata = HeadObjectResponse.builder()
61+
.build();
62+
63+
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - expirationTimeRuleId - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.expirationTimeRuleId("expirationTimeRuleId");
64+
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - ongoingRestore - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.ongoingRestore(false);
65+
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - requesterCharged - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.requesterCharged(false);
66+
67+
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - lastModified - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.lastModified(dateVal);
68+
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - expirationTime - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.expirationTime(dateVal);
69+
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - restoreExpirationTime - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.restoreExpirationTime(dateVal);
70+
71+
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - header - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.header("key", "val");
72+
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - addUserMetadata - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.addUserMetadata("a", "b");
3773
}
3874

3975
private void generatePresignedUrl(S3Client s3, String bucket, String key, Date expiration) {
@@ -43,7 +79,6 @@ private void generatePresignedUrl(S3Client s3, String bucket, String key, Date e
4379

4480
URL urlPost = /*AWS SDK for Java v2 migration: S3 generatePresignedUrl() with POST HTTP method is not supported in v2. Only GET, PUT, and DELETE are supported - https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/presigner/S3Presigner.html*/s3.generatePresignedUrl(bucket, key, expiration, HttpMethod.POST);
4581

46-
4782
HttpMethod httpMethod = HttpMethod.PUT;
4883
URL urlWithHttpMethodVariable = /*AWS SDK for Java v2 migration: Transform for S3 generatePresignedUrl() with an assigned variable for HttpMethod is not supported. Please manually migrate your code - https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/presigner/S3Presigner.html*/s3.generatePresignedUrl(bucket, key, expiration, httpMethod);
4984

‎test/v2-migration-tests/src/test/resources/software/amazon/awssdk/v2migrationtests/maven-nocompile/before/src/main/java/foo/bar/S3Transforms.java

+39-5
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import com.amazonaws.HttpMethod;
1919
import com.amazonaws.services.s3.AmazonS3;
2020
import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
21+
import com.amazonaws.services.s3.model.ObjectMetadata;
2122
import com.amazonaws.services.s3.model.PutObjectRequest;
2223
import com.amazonaws.services.s3.transfer.TransferManager;
2324
import java.io.ByteArrayInputStream;
@@ -27,11 +28,45 @@
2728

2829
public class S3Transforms {
2930

30-
void upload(TransferManager tm, String bucket, String key) {
31+
void upload_streamWithLiteralLength(TransferManager tm, String bucket, String key) {
32+
ObjectMetadata metadata = new ObjectMetadata();
33+
metadata.setContentLength(333);
3134
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
32-
PutObjectRequest requestWithInputStream = new PutObjectRequest(bucket, key, "location");
33-
requestWithInputStream.setInputStream(inputStream);
34-
tm.upload(requestWithInputStream);
35+
PutObjectRequest requestWithStreamAndLiteralLength = new PutObjectRequest(bucket, key, "location").withMetadata(metadata);
36+
requestWithStreamAndLiteralLength.setInputStream(inputStream);
37+
tm.upload(requestWithStreamAndLiteralLength);
38+
}
39+
40+
void upload_streamWithAssignedLength(TransferManager tm, String bucket, String key) {
41+
ObjectMetadata metadata = new ObjectMetadata();
42+
long contentLen = 777;
43+
metadata.setContentLength(contentLen);
44+
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
45+
PutObjectRequest requestWithStreamAndAssignedLength = new PutObjectRequest(bucket, key, "location").withMetadata(metadata);
46+
requestWithStreamAndAssignedLength.setInputStream(inputStream);
47+
tm.upload(requestWithStreamAndAssignedLength);
48+
}
49+
50+
void upload_streamWithoutLength(TransferManager tm, String bucket, String key) {
51+
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
52+
PutObjectRequest requestWithStreamAndNoLength = new PutObjectRequest(bucket, key, "location");
53+
requestWithStreamAndNoLength.setInputStream(inputStream);
54+
tm.upload(requestWithStreamAndNoLength);
55+
}
56+
57+
void objectmetadata_unsupportedSetters(Date dateVal) {
58+
ObjectMetadata metadata = new ObjectMetadata();
59+
60+
metadata.setExpirationTimeRuleId("expirationTimeRuleId");
61+
metadata.setOngoingRestore(false);
62+
metadata.setRequesterCharged(false);
63+
64+
metadata.setLastModified(dateVal);
65+
metadata.setExpirationTime(dateVal);
66+
metadata.setRestoreExpirationTime(dateVal);
67+
68+
metadata.setHeader("key", "val");
69+
metadata.addUserMetadata("a", "b");
3570
}
3671

3772
private void generatePresignedUrl(AmazonS3 s3, String bucket, String key, Date expiration) {
@@ -41,7 +76,6 @@ private void generatePresignedUrl(AmazonS3 s3, String bucket, String key, Date e
4176

4277
URL urlPost = s3.generatePresignedUrl(bucket, key, expiration, HttpMethod.POST);
4378

44-
4579
HttpMethod httpMethod = HttpMethod.PUT;
4680
URL urlWithHttpMethodVariable = s3.generatePresignedUrl(bucket, key, expiration, httpMethod);
4781

‎test/v2-migration-tests/src/test/resources/software/amazon/awssdk/v2migrationtests/maven/after/src/main/java/foo/bar/S3Streaming.java

+70-3
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,15 @@
1818
import java.io.ByteArrayInputStream;
1919
import java.io.File;
2020
import java.io.InputStream;
21+
import java.util.Date;
22+
import java.util.HashMap;
23+
import java.util.Map;
2124
import software.amazon.awssdk.core.ResponseInputStream;
2225
import software.amazon.awssdk.core.sync.RequestBody;
2326
import software.amazon.awssdk.services.s3.S3Client;
2427
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
2528
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
29+
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
2630
import software.amazon.awssdk.services.s3.model.ObjectCannedACL;
2731
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
2832
import software.amazon.awssdk.services.s3.model.RequestPayer;
@@ -47,6 +51,18 @@ void putObject_bucketKeyFile(String bucket, String key, File file) {
4751
.build(), RequestBody.fromFile(file));
4852
}
4953

54+
void putObject_bucketKeyStreamMetadata(String bucket, String key, InputStream stream) {
55+
HeadObjectResponse metadataWithLength = HeadObjectResponse.builder()
56+
.build();
57+
s3.putObject(PutObjectRequest.builder().bucket(bucket).key(key).contentLength(22L)
58+
.build(), RequestBody.fromInputStream(stream, 22L));
59+
60+
61+
HeadObjectResponse metadataWithoutLength = HeadObjectResponse.builder()
62+
.build();
63+
/*AWS SDK for Java v2 migration: When using InputStream to upload with S3Client, Content-Length should be specified and used with RequestBody.fromInputStream(). Otherwise, the entire stream will be buffered in memory. If content length must be unknown, we recommend using the CRT-based S3 client - https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/crt-based-s3-client.html*/s3.putObject(PutObjectRequest.builder().bucket(bucket).key(key).build(), RequestBody.fromContentProvider(() -> stream, "application/octet-stream"));
64+
}
65+
5066
/**
5167
* Mixed ordering to ensure the files are assigned correctly
5268
*/
@@ -76,10 +92,12 @@ void putObject_requestPojoWithInputStream(String bucket, String key) {
7692

7793
PutObjectRequest request1 = PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location")
7894
.build();
79-
/*AWS SDK for Java v2 migration: When using InputStream to upload with S3Client, Content-Length should be specified and used with RequestBody.fromInputStream(). Otherwise, the entire stream will be buffered in memory.*/s3.putObject(request1, RequestBody.fromContentProvider(() -> inputStream1, "binary/octet-stream"));
95+
/*AWS SDK for Java v2 migration: When using InputStream to upload with S3Client, Content-Length should be specified and used with RequestBody.fromInputStream(). Otherwise, the entire stream will be buffered in memory. If content length must be unknown, we recommend using the CRT-based S3 client - https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/crt-based-s3-client.html*/s3.putObject(request1, RequestBody.fromContentProvider(() -> inputStream1, "application/octet-stream"));
8096

81-
/*AWS SDK for Java v2 migration: When using InputStream to upload with S3Client, Content-Length should be specified and used with RequestBody.fromInputStream(). Otherwise, the entire stream will be buffered in memory.*/s3.putObject(PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location")
82-
.build(), RequestBody.fromContentProvider(() -> inputStream2, "binary/octet-stream"));
97+
HeadObjectResponse metadata = HeadObjectResponse.builder()
98+
.build();
99+
s3.putObject(PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location").contentLength(11L)
100+
.build(), RequestBody.fromInputStream(inputStream2, 11L));
83101
}
84102

85103
void putObject_requestPojoWithoutPayload(String bucket, String key) {
@@ -104,4 +122,53 @@ void putObjectRequesterPaysSetter() {
104122
PutObjectRequest requestWithFalse =PutObjectRequest.builder().bucket("bucket").key("key").websiteRedirectLocation("location")
105123
.build();
106124
}
125+
126+
void putObjectRequest_setMetadata() {
127+
HeadObjectResponse metadata = HeadObjectResponse.builder()
128+
.build();
129+
130+
PutObjectRequest request = PutObjectRequest.builder().bucket("bucket").key("key").websiteRedirectLocation("location")
131+
.build();
132+
request = request.toBuilder().contentLength(66L)
133+
.contentEncoding("UTF-8")
134+
.contentType("text/plain")
135+
.build();
136+
}
137+
138+
void putObjectRequest_withMetadata() {
139+
HeadObjectResponse metadata = HeadObjectResponse.builder()
140+
.build();
141+
long contentLen = 66;
142+
Date expiry = new Date();
143+
144+
Map<String, String> userMetadata = new HashMap<>();
145+
userMetadata.put("key", "value");
146+
147+
PutObjectRequest request = PutObjectRequest.builder().bucket("bucket").key("key").websiteRedirectLocation("location").contentLength(contentLen)
148+
.contentEncoding("UTF-8")
149+
.contentType("text/plain")
150+
.contentLanguage("en-US")
151+
.cacheControl("must-revalidate")
152+
.contentDisposition("inline")
153+
.contentMD5("md5Val")
154+
.serverSideEncryption("sseEncryptionVal")
155+
.serverSideEncryption("sseAlgorithmVal")
156+
.sseCustomerKeyMD5("sseCustomerKeyMd5Val")
157+
.bucketKeyEnabled(true)
158+
.metadata(userMetadata)
159+
.expires(expiry.toInstant())
160+
.build();
161+
}
162+
163+
void putObjectRequester_emptyMetadata() {
164+
HeadObjectResponse emptyMetadata1 = HeadObjectResponse.builder()
165+
.build();
166+
PutObjectRequest request1 =PutObjectRequest.builder().bucket("bucket").key("key").websiteRedirectLocation("location")
167+
.build();
168+
169+
HeadObjectResponse emptyMetadata2 = HeadObjectResponse.builder()
170+
.build();
171+
PutObjectRequest request2 = PutObjectRequest.builder().bucket("bucket").key("key").websiteRedirectLocation("location")
172+
.build();
173+
}
107174
}

‎test/v2-migration-tests/src/test/resources/software/amazon/awssdk/v2migrationtests/maven/before/src/main/java/foo/bar/S3Streaming.java

+60-1
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,15 @@
1818
import com.amazonaws.services.s3.AmazonS3;
1919
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
2020
import com.amazonaws.services.s3.model.CannedAccessControlList;
21+
import com.amazonaws.services.s3.model.ObjectMetadata;
2122
import com.amazonaws.services.s3.model.PutObjectRequest;
2223
import com.amazonaws.services.s3.model.S3Object;
2324
import java.io.ByteArrayInputStream;
2425
import java.io.File;
2526
import java.io.InputStream;
27+
import java.util.Date;
28+
import java.util.HashMap;
29+
import java.util.Map;
2630

2731
public class S3Streaming {
2832

@@ -41,6 +45,16 @@ void putObject_bucketKeyFile(String bucket, String key, File file) {
4145
s3.putObject(bucket, key, file);
4246
}
4347

48+
void putObject_bucketKeyStreamMetadata(String bucket, String key, InputStream stream) {
49+
ObjectMetadata metadataWithLength = new ObjectMetadata();
50+
metadataWithLength.setContentLength(22);
51+
s3.putObject(bucket, key, stream, metadataWithLength);
52+
53+
54+
ObjectMetadata metadataWithoutLength = new ObjectMetadata();
55+
s3.putObject(bucket, key, stream, metadataWithoutLength);
56+
}
57+
4458
/**
4559
* Mixed ordering to ensure the files are assigned correctly
4660
*/
@@ -69,7 +83,9 @@ void putObject_requestPojoWithInputStream(String bucket, String key) {
6983
request1.setInputStream(inputStream1);
7084
s3.putObject(request1);
7185

72-
s3.putObject(new PutObjectRequest(bucket, key, "location").withInputStream(inputStream2));
86+
ObjectMetadata metadata = new ObjectMetadata();
87+
metadata.setContentLength(11);
88+
s3.putObject(new PutObjectRequest(bucket, key, "location").withInputStream(inputStream2).withMetadata(metadata));
7389
}
7490

7591
void putObject_requestPojoWithoutPayload(String bucket, String key) {
@@ -90,4 +106,47 @@ void putObjectRequesterPaysSetter() {
90106

91107
PutObjectRequest requestWithFalse = new PutObjectRequest("bucket", "key", "location").withRequesterPays(false);
92108
}
109+
110+
void putObjectRequest_setMetadata() {
111+
ObjectMetadata metadata = new ObjectMetadata();
112+
metadata.setContentLength(66);
113+
metadata.setContentType("text/plain");
114+
metadata.setContentEncoding("UTF-8");
115+
116+
PutObjectRequest request = new PutObjectRequest("bucket", "key", "location");
117+
request.setMetadata(metadata);
118+
}
119+
120+
void putObjectRequest_withMetadata() {
121+
ObjectMetadata metadata = new ObjectMetadata();
122+
long contentLen = 66;
123+
metadata.setContentLength(contentLen);
124+
metadata.setContentType("text/plain");
125+
metadata.setContentEncoding("UTF-8");
126+
metadata.setContentLanguage("en-US");
127+
metadata.setCacheControl("must-revalidate");
128+
metadata.setContentDisposition("inline");
129+
metadata.setContentMD5("md5Val");
130+
metadata.setSSEAlgorithm("sseAlgorithmVal");
131+
metadata.setServerSideEncryption("sseEncryptionVal");
132+
metadata.setSSECustomerKeyMd5("sseCustomerKeyMd5Val");
133+
metadata.setBucketKeyEnabled(true);
134+
Date expiry = new Date();
135+
metadata.setHttpExpiresDate(expiry);
136+
137+
Map<String, String> userMetadata = new HashMap<>();
138+
userMetadata.put("key", "value");
139+
metadata.setUserMetadata(userMetadata);
140+
141+
PutObjectRequest request = new PutObjectRequest("bucket", "key", "location").withMetadata(metadata);
142+
}
143+
144+
void putObjectRequester_emptyMetadata() {
145+
ObjectMetadata emptyMetadata1 = new ObjectMetadata();
146+
PutObjectRequest request1 = new PutObjectRequest("bucket", "key", "location").withMetadata(emptyMetadata1);
147+
148+
ObjectMetadata emptyMetadata2 = new ObjectMetadata();
149+
PutObjectRequest request2 = new PutObjectRequest("bucket", "key", "location");
150+
request2.setMetadata(emptyMetadata2);
151+
}
93152
}

0 commit comments

Comments
 (0)