Skip to content

Commit 203e5dc

Browse files
imabhichowimabhichow
authored
feat: multipart & ranged get examples (#168)
1 parent 5502eab commit 203e5dc

File tree

7 files changed

+471
-0
lines changed

7 files changed

+471
-0
lines changed

‎src/examples/java/software/amazon/encryption/s3/examples/AsyncClientExample.java

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package software.amazon.encryption.s3.examples;
2+
3+
import static org.junit.jupiter.api.Assertions.assertEquals;
4+
import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.KMS_KEY_ID;
5+
import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.appendTestSuffix;
6+
7+
import software.amazon.awssdk.core.ResponseBytes;
8+
import software.amazon.awssdk.core.async.AsyncRequestBody;
9+
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
10+
import software.amazon.awssdk.services.s3.S3AsyncClient;
11+
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
12+
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
13+
import software.amazon.encryption.s3.S3AsyncEncryptionClient;
14+
15+
import java.util.concurrent.CompletableFuture;
16+
17+
public class AsyncClientExample {
18+
public static final String OBJECT_KEY = appendTestSuffix("async-client-example");
19+
20+
public static void main(final String[] args) {
21+
String bucket = args[0];
22+
AsyncClient(bucket);
23+
cleanup(bucket);
24+
}
25+
26+
/**
27+
* This example demonstrates handling of an Asynchronous S3 Encryption Client for interacting with an Amazon S3 bucket, performing
28+
* both encryption and decryption using the AWS KMS Key Arn for secure object storage.
29+
*
30+
* @param bucket The name of the Amazon S3 bucket to perform operations on.
31+
*/
32+
public static void AsyncClient(String bucket) {
33+
final String input = "PutAsyncGetAsync";
34+
35+
// Instantiate the S3 Async Encryption Client to encrypt and decrypt
36+
// by specifying an AES Key with the aesKey builder parameter.
37+
//
38+
// This means that the S3 Async Encryption Client can perform both encrypt and decrypt operations
39+
// as part of the S3 putObject and getObject operations.
40+
S3AsyncClient v3AsyncClient = S3AsyncEncryptionClient.builder()
41+
.kmsKeyId(KMS_KEY_ID)
42+
.build();
43+
44+
// Call putObject to encrypt the object and upload it to S3
45+
CompletableFuture<PutObjectResponse> futurePut = v3AsyncClient.putObject(builder -> builder
46+
.bucket(bucket)
47+
.key(OBJECT_KEY)
48+
.build(), AsyncRequestBody.fromString(input));
49+
// Block on completion of the futurePut
50+
futurePut.join();
51+
52+
// Call getObject to retrieve and decrypt the object from S3
53+
CompletableFuture<ResponseBytes<GetObjectResponse>> futureGet = v3AsyncClient.getObject(builder -> builder
54+
.bucket(bucket)
55+
.key(OBJECT_KEY)
56+
.build(), AsyncResponseTransformer.toBytes());
57+
// Just wait for the future to complete
58+
ResponseBytes<GetObjectResponse> getResponse = futureGet.join();
59+
60+
// Assert
61+
assertEquals(input, getResponse.asUtf8String());
62+
63+
// Close the client
64+
v3AsyncClient.close();
65+
}
66+
67+
private static void cleanup(String bucket) {
68+
// Instantiate the client to delete object
69+
S3AsyncClient v3Client = S3AsyncEncryptionClient.builder()
70+
.kmsKeyId(KMS_KEY_ID)
71+
.build();
72+
73+
// Call deleteObject to delete the object from given S3 Bucket
74+
v3Client.deleteObject(builder -> builder.bucket(bucket)
75+
.key(OBJECT_KEY)).join();
76+
77+
// Close the client
78+
v3Client.close();
79+
}
80+
}

‎src/examples/java/software/amazon/encryption/s3/examples/MultipartUploadExample.java

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
package software.amazon.encryption.s3.examples;
2+
3+
import static org.junit.jupiter.api.Assertions.assertTrue;
4+
import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration;
5+
import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.KMS_KEY_ID;
6+
import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.appendTestSuffix;
7+
8+
import software.amazon.awssdk.core.ResponseInputStream;
9+
import software.amazon.awssdk.core.sync.RequestBody;
10+
import software.amazon.awssdk.services.s3.S3Client;
11+
import software.amazon.awssdk.services.s3.model.CompletedPart;
12+
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
13+
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
14+
import software.amazon.awssdk.services.s3.model.SdkPartType;
15+
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
16+
import software.amazon.awssdk.services.s3.model.UploadPartResponse;
17+
import software.amazon.encryption.s3.S3EncryptionClient;
18+
import software.amazon.encryption.s3.utils.BoundedInputStream;
19+
20+
import java.io.ByteArrayInputStream;
21+
import java.io.ByteArrayOutputStream;
22+
import java.io.IOException;
23+
import java.io.InputStream;
24+
import java.util.ArrayList;
25+
import java.util.HashMap;
26+
import java.util.List;
27+
import java.util.Map;
28+
29+
import org.apache.commons.io.IOUtils;
30+
31+
public class MultipartUploadExample {
32+
public static String BUCKET;
33+
public static void main(final String[] args) throws IOException {
34+
BUCKET = args[0];
35+
LowLevelMultipartUpload();
36+
HighLevelMultipartPutObject();
37+
}
38+
39+
/**
40+
* This example demonstrates a low-level approach to performing a multipart upload to an Amazon S3 bucket
41+
* using the S3 Client, performing encryption and decryption operations using the AWS Key Management Service (KMS).
42+
* It showcases the process of uploading a large object in smaller parts, processing and encrypting each part,
43+
* and then completing the multipart upload.
44+
*
45+
* @throws IOException If an I/O error occurs while reading or writing data.
46+
*/
47+
public static void LowLevelMultipartUpload() throws IOException {
48+
final String objectKey = appendTestSuffix("low-level-multipart-upload-example");
49+
50+
// Overall "file" is 100MB, split into 10MB parts
51+
final long fileSizeLimit = 1024 * 1024 * 100;
52+
final int PART_SIZE = 10 * 1024 * 1024;
53+
final InputStream inputStream = new BoundedInputStream(fileSizeLimit);
54+
final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit);
55+
56+
// Instantiate the S3 Encryption Client to encrypt and decrypt
57+
// by specifying a KMS Key with the kmsKeyId builder parameter.
58+
// enable `enableDelayedAuthenticationMode` parameter to download more than 64MB object or
59+
// configure buffer size using `setBufferSize()` to increase your default buffer size.
60+
//
61+
// This means that the S3 Encryption Client can perform both encrypt and decrypt operations
62+
// as part of the S3 putObject and getObject operations.
63+
S3Client v3Client = S3EncryptionClient.builder()
64+
.kmsKeyId(KMS_KEY_ID)
65+
.enableDelayedAuthenticationMode(true)
66+
.build();
67+
68+
// Create Multipart upload request to S3
69+
CreateMultipartUploadResponse initiateResult = v3Client.createMultipartUpload(builder ->
70+
builder.bucket(BUCKET).key(objectKey));
71+
72+
List<CompletedPart> partETags = new ArrayList<>();
73+
74+
int bytesRead, bytesSent = 0;
75+
// 10MB parts
76+
byte[] partData = new byte[PART_SIZE];
77+
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
78+
int partsSent = 1;
79+
80+
// Process and Upload each part
81+
while ((bytesRead = inputStream.read(partData, 0, partData.length)) != -1) {
82+
outputStream.write(partData, 0, bytesRead);
83+
if (bytesSent < PART_SIZE) {
84+
bytesSent += bytesRead;
85+
continue;
86+
}
87+
88+
// Create UploadPartRequest for each part by specifying partNumber and
89+
// set isLastPart as true only if the part is the last remaining part in the multipart upload.
90+
UploadPartRequest uploadPartRequest = UploadPartRequest.builder()
91+
.bucket(BUCKET)
92+
.key(objectKey)
93+
.uploadId(initiateResult.uploadId())
94+
.partNumber(partsSent)
95+
.build();
96+
97+
final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray());
98+
99+
// Upload all the different parts of the object
100+
UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest,
101+
RequestBody.fromInputStream(partInputStream, partInputStream.available()));
102+
103+
// We need to add eTag's of all CompletedParts before calling CompleteMultipartUpload.
104+
partETags.add(CompletedPart.builder()
105+
.partNumber(partsSent)
106+
.eTag(uploadPartResult.eTag())
107+
.build());
108+
109+
outputStream.reset();
110+
bytesSent = 0;
111+
partsSent++;
112+
}
113+
inputStream.close();
114+
115+
// Set sdkPartType to SdkPartType.LAST for last part of the multipart upload.
116+
//
117+
// Note: Set sdkPartType parameter to SdkPartType.LAST for last part is required for Multipart Upload in S3EncryptionClient to call `cipher.doFinal()`
118+
UploadPartRequest uploadPartRequest = UploadPartRequest.builder()
119+
.bucket(BUCKET)
120+
.key(objectKey)
121+
.uploadId(initiateResult.uploadId())
122+
.partNumber(partsSent)
123+
.sdkPartType(SdkPartType.LAST)
124+
.build();
125+
126+
// Upload the last part multipart upload to invoke `cipher.doFinal()`
127+
final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray());
128+
UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest,
129+
RequestBody.fromInputStream(partInputStream, partInputStream.available()));
130+
131+
partETags.add(CompletedPart.builder()
132+
.partNumber(partsSent)
133+
.eTag(uploadPartResult.eTag())
134+
.build());
135+
136+
// Finally call completeMultipartUpload operation to tell S3 to merge all uploaded
137+
// parts and finish the multipart operation.
138+
v3Client.completeMultipartUpload(builder -> builder
139+
.bucket(BUCKET)
140+
.key(objectKey)
141+
.uploadId(initiateResult.uploadId())
142+
.multipartUpload(partBuilder -> partBuilder.parts(partETags)));
143+
144+
// Call getObject to retrieve and decrypt the object from S3
145+
ResponseInputStream<GetObjectResponse> output = v3Client.getObject(builder -> builder
146+
.bucket(BUCKET)
147+
.key(objectKey));
148+
149+
// Asserts
150+
assertTrue(IOUtils.contentEquals(objectStreamForResult, output));
151+
152+
// Cleanup
153+
v3Client.deleteObject(builder -> builder.bucket(BUCKET).key(objectKey));
154+
v3Client.close();
155+
}
156+
157+
/**
158+
* This example demonstrates a high-level approach to performing a multipart object upload to an Amazon S3 bucket
159+
* using the S3 Client, performing encryption and decryption operations using the AWS KMS Key Arn.
160+
* This allows faster putObject by creating an on-disk encrypted copy before uploading to S3.
161+
*
162+
* @throws IOException If an I/O error occurs while reading or writing data.
163+
*/
164+
public static void HighLevelMultipartPutObject() throws IOException {
165+
final String objectKey = appendTestSuffix("high-level-multipart-upload-example");
166+
167+
// Overall "file" is 100MB, split into 10MB parts
168+
final long fileSizeLimit = 1024 * 1024 * 100;
169+
final InputStream inputStream = new BoundedInputStream(fileSizeLimit);
170+
final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit);
171+
172+
// Instantiate the S3 Encryption Client to encrypt and decrypt
173+
// by specifying a KMS Key with the kmsKeyId builder parameter.
174+
// enable `enableMultipartPutObject` allows faster putObject by creating an on-disk encrypted copy before uploading to S3.
175+
//
176+
// This means that the S3 Encryption Client can perform both encrypt and decrypt operations
177+
// as part of the S3 putObject and getObject operations.
178+
// Note: You must also specify the `enableDelayedAuthenticationMode` parameter to perform getObject with more than 64MB object.
179+
S3Client v3Client = S3EncryptionClient.builder()
180+
.kmsKeyId(KMS_KEY_ID)
181+
.enableMultipartPutObject(true)
182+
.enableDelayedAuthenticationMode(true)
183+
.build();
184+
185+
// Create an encryption context
186+
Map<String, String> encryptionContext = new HashMap<>();
187+
encryptionContext.put("user-metadata-key", "user-metadata-value-v3-to-v3");
188+
189+
// Call putObject to encrypt the object and upload it to S3.
190+
v3Client.putObject(builder -> builder
191+
.bucket(BUCKET)
192+
.overrideConfiguration(withAdditionalConfiguration(encryptionContext))
193+
.key(objectKey), RequestBody.fromInputStream(inputStream, fileSizeLimit));
194+
195+
// Call getObject to retrieve and decrypt the object from S3.
196+
ResponseInputStream<GetObjectResponse> output = v3Client.getObject(builder -> builder
197+
.bucket(BUCKET)
198+
.overrideConfiguration(withAdditionalConfiguration(encryptionContext))
199+
.key(objectKey));
200+
201+
// Asserts
202+
assertTrue(IOUtils.contentEquals(objectStreamForResult, output));
203+
204+
// Cleanup
205+
v3Client.deleteObject(builder -> builder.bucket(BUCKET).key(objectKey));
206+
v3Client.close();
207+
}
208+
}

0 commit comments

Comments
 (0)