Skip to content

Commit a094c4b

Browse files
ShallwewuShallwewu
authored
Merge pull request #177 from wujinhu/master
 oss select sdk support json format
2 parents edfffa8 + 28d3af0 commit a094c4b

16 files changed

+563
-89
lines changed

‎src/main/java/com/aliyun/oss/common/parser/RequestMarshallers.java‎

Lines changed: 97 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -482,20 +482,33 @@ public static final class CreateSelectObjectMetadataRequestMarshaller
482482

483483
@Override
484484
public byte[] marshall(CreateSelectObjectMetadataRequest request) {
485-
StringBuffer xmlBody = new StringBuffer();
485+
StringBuilder xmlBody = new StringBuilder();
486486
InputSerialization inputSerialization = request.getInputSerialization();
487487
CSVFormat csvFormat = inputSerialization.getCsvInputFormat();
488-
xmlBody.append("<CsvMetaRequest>");
489-
xmlBody.append("<InputSerialization>");
490-
xmlBody.append("<CompressionType>" + inputSerialization.getCompressionType() + "</CompressionType>");
491-
xmlBody.append("<CSV>");
492-
xmlBody.append("<RecordDelimiter>" + BinaryUtil.toBase64String(csvFormat.getRecordDelimiter().getBytes()) + "</RecordDelimiter>");
493-
xmlBody.append("<FieldDelimiter>" + BinaryUtil.toBase64String(csvFormat.getFieldDelimiter().toString().getBytes()) + "</FieldDelimiter>");
494-
xmlBody.append("<QuoteCharacter>" + BinaryUtil.toBase64String(csvFormat.getQuoteChar().toString().getBytes()) + "</QuoteCharacter>");
495-
xmlBody.append("</CSV>");
496-
xmlBody.append("</InputSerialization>");
497-
xmlBody.append("<OverwriteIfExists>" + request.isOverwrite() + "</OverwriteIfExists>");
498-
xmlBody.append("</CsvMetaRequest>");
488+
JsonFormat jsonFormat = inputSerialization.getJsonInputFormat();
489+
if (inputSerialization.getSelectContentFormat() == SelectContentFormat.CSV) {
490+
xmlBody.append("<CsvMetaRequest>");
491+
xmlBody.append("<InputSerialization>");
492+
xmlBody.append("<CompressionType>" + inputSerialization.getCompressionType() + "</CompressionType>");
493+
xmlBody.append("<CSV>");
494+
xmlBody.append("<RecordDelimiter>" + BinaryUtil.toBase64String(csvFormat.getRecordDelimiter().getBytes()) + "</RecordDelimiter>");
495+
xmlBody.append("<FieldDelimiter>" + BinaryUtil.toBase64String(csvFormat.getFieldDelimiter().toString().getBytes()) + "</FieldDelimiter>");
496+
xmlBody.append("<QuoteCharacter>" + BinaryUtil.toBase64String(csvFormat.getQuoteChar().toString().getBytes()) + "</QuoteCharacter>");
497+
xmlBody.append("</CSV>");
498+
xmlBody.append("</InputSerialization>");
499+
xmlBody.append("<OverwriteIfExists>" + request.isOverwrite() + "</OverwriteIfExists>");
500+
xmlBody.append("</CsvMetaRequest>");
501+
} else {
502+
xmlBody.append("<JsonMetaRequest>");
503+
xmlBody.append("<InputSerialization>");
504+
xmlBody.append("<CompressionType>" + inputSerialization.getCompressionType() + "</CompressionType>");
505+
xmlBody.append("<JSON>");
506+
xmlBody.append("<Type>" + jsonFormat.getJsonType().name() + "</Type>");
507+
xmlBody.append("</JSON>");
508+
xmlBody.append("</InputSerialization>");
509+
xmlBody.append("<OverwriteIfExists>" + request.isOverwrite() + "</OverwriteIfExists>");
510+
xmlBody.append("</JsonMetaRequest>");
511+
}
499512

500513
try {
501514
return xmlBody.toString().getBytes(DEFAULT_CHARSET_NAME);
@@ -505,49 +518,89 @@ public byte[] marshall(CreateSelectObjectMetadataRequest request) {
505518
}
506519
}
507520

521+
private static void populateSelectRange(StringBuilder xmlBody, SelectObjectRequest request) {
522+
if (request.getLineRange() != null) {
523+
xmlBody.append("<Range>" + request.lineRangeToString(request.getLineRange()) + "</Range>");
524+
}
525+
if (request.getSplitRange() != null) {
526+
xmlBody.append("<Range>" + request.splitRangeToString(request.getSplitRange()) + "</Range>");
527+
}
528+
}
529+
530+
private static void populateSelectJsonObjectRequest(StringBuilder xmlBody, SelectObjectRequest request) {
531+
InputSerialization inputSerialization = request.getInputSerialization();
532+
JsonFormat jsonInputFormat = inputSerialization.getJsonInputFormat();
533+
xmlBody.append("<InputSerialization>");
534+
xmlBody.append("<CompressionType>" + inputSerialization.getCompressionType() + "</CompressionType>");
535+
xmlBody.append("<JSON>");
536+
xmlBody.append("<Type>" + jsonInputFormat.getJsonType().name() + "</Type>");
537+
xmlBody.append("<ParseJsonNumberAsString>" + jsonInputFormat.isParseJsonNumberAsString() + "</ParseJsonNumberAsString>");
538+
populateSelectRange(xmlBody, request);
539+
xmlBody.append("</JSON>");
540+
xmlBody.append("</InputSerialization>");
541+
542+
OutputSerialization outputSerialization = request.getOutputSerialization();
543+
xmlBody.append("<OutputSerialization>");
544+
xmlBody.append("<JSON>");
545+
xmlBody.append("<RecordDelimiter>" + BinaryUtil.toBase64String(outputSerialization.getJsonOutputFormat().getRecordDelimiter().getBytes()) + "</RecordDelimiter>");
546+
xmlBody.append("</JSON>");
547+
xmlBody.append("<OutputRawData>" + outputSerialization.isOutputRawData() + "</OutputRawData>");
548+
xmlBody.append("<EnablePayloadCrc>" + outputSerialization.isPayloadCrcEnabled() + "</EnablePayloadCrc>");
549+
xmlBody.append("</OutputSerialization>");
550+
}
551+
552+
private static void populateSelectCsvObjectRequest(StringBuilder xmlBody, SelectObjectRequest request) {
553+
InputSerialization inputSerialization = request.getInputSerialization();
554+
CSVFormat csvInputFormat = inputSerialization.getCsvInputFormat();
555+
xmlBody.append("<InputSerialization>");
556+
xmlBody.append("<CompressionType>" + inputSerialization.getCompressionType() + "</CompressionType>");
557+
xmlBody.append("<CSV>");
558+
xmlBody.append("<FileHeaderInfo>" + csvInputFormat.getHeaderInfo() + "</FileHeaderInfo>");
559+
xmlBody.append("<RecordDelimiter>" + BinaryUtil.toBase64String(csvInputFormat.getRecordDelimiter().getBytes()) + "</RecordDelimiter>");
560+
xmlBody.append("<FieldDelimiter>" + BinaryUtil.toBase64String(csvInputFormat.getFieldDelimiter().toString().getBytes()) + "</FieldDelimiter>");
561+
xmlBody.append("<QuoteCharacter>" + BinaryUtil.toBase64String(csvInputFormat.getQuoteChar().toString().getBytes()) + "</QuoteCharacter>");
562+
xmlBody.append("<CommentCharacter>" + BinaryUtil.toBase64String(csvInputFormat.getCommentChar().toString().getBytes()) + "</CommentCharacter>");
563+
populateSelectRange(xmlBody, request);
564+
xmlBody.append("</CSV>");
565+
xmlBody.append("</InputSerialization>");
566+
567+
OutputSerialization outputSerialization = request.getOutputSerialization();
568+
xmlBody.append("<OutputSerialization>");
569+
xmlBody.append("<CSV>");
570+
xmlBody.append("<RecordDelimiter>" + BinaryUtil.toBase64String(outputSerialization.getCsvOutputFormat().getRecordDelimiter().getBytes()) + "</RecordDelimiter>");
571+
xmlBody.append("<FieldDelimiter>" + BinaryUtil.toBase64String(outputSerialization.getCsvOutputFormat().getFieldDelimiter().toString().getBytes()) + "</FieldDelimiter>");
572+
xmlBody.append("<QuoteCharacter>" + BinaryUtil.toBase64String(outputSerialization.getCsvOutputFormat().getQuoteChar().toString().getBytes()) + "</QuoteCharacter>");
573+
xmlBody.append("</CSV>");
574+
xmlBody.append("<KeepAllColumns>" + outputSerialization.isKeepAllColumns() + "</KeepAllColumns>");
575+
xmlBody.append("<OutputHeader>" + outputSerialization.isOutputHeader() + "</OutputHeader>");
576+
xmlBody.append("<OutputRawData>" + outputSerialization.isOutputRawData() + "</OutputRawData>");
577+
xmlBody.append("<EnablePayloadCrc>" + outputSerialization.isPayloadCrcEnabled() + "</EnablePayloadCrc>");
578+
xmlBody.append("</OutputSerialization>");
579+
}
580+
508581
public static final class SelectObjectRequestMarshaller implements RequestMarshaller2<SelectObjectRequest> {
509582

510583
@Override
511584
public byte[] marshall(SelectObjectRequest request) {
512-
StringBuffer xmlBody = new StringBuffer();
585+
StringBuilder xmlBody = new StringBuilder();
513586
xmlBody.append("<SelectRequest>");
514587

515588
xmlBody.append("<Expression>" + BinaryUtil.toBase64String(request.getExpression().getBytes()) + "</Expression>");
516589
xmlBody.append("<Options>");
517590
xmlBody.append("<SkipPartialDataRecord>" + request.isSkipPartialDataRecord() + "</SkipPartialDataRecord>");
591+
if (request.getMaxSkippedRecordsAllowed() > 0) {
592+
xmlBody.append("<MaxSkippedRecordsAllowed>" + request.getMaxSkippedRecordsAllowed() + "</MaxSkippedRecordsAllowed>");
593+
}
518594
xmlBody.append("</Options>");
519595
InputSerialization inputSerialization = request.getInputSerialization();
520-
CSVFormat csvInputFormat = inputSerialization.getCsvInputFormat();
521-
xmlBody.append("<InputSerialization>");
522-
xmlBody.append("<CompressionType>" + inputSerialization.getCompressionType() + "</CompressionType>");
523-
xmlBody.append("<CSV>");
524-
xmlBody.append("<FileHeaderInfo>" + csvInputFormat.getHeaderInfo() + "</FileHeaderInfo>");
525-
xmlBody.append("<RecordDelimiter>" + BinaryUtil.toBase64String(csvInputFormat.getRecordDelimiter().getBytes()) + "</RecordDelimiter>");
526-
xmlBody.append("<FieldDelimiter>" + BinaryUtil.toBase64String(csvInputFormat.getFieldDelimiter().toString().getBytes()) + "</FieldDelimiter>");
527-
xmlBody.append("<QuoteCharacter>" + BinaryUtil.toBase64String(csvInputFormat.getQuoteChar().toString().getBytes()) + "</QuoteCharacter>");
528-
xmlBody.append("<CommentCharacter>" + BinaryUtil.toBase64String(csvInputFormat.getCommentChar().toString().getBytes()) + "</CommentCharacter>");
529-
530-
if (request.getLineRange() != null) {
531-
xmlBody.append("<Range>" + request.lineRangeToString(request.getLineRange()) + "</Range>");
532-
}
533-
if (request.getSplitRange() != null) {
534-
xmlBody.append("<Range>" + request.splitRangeToString(request.getSplitRange()) + "</Range>");
535-
}
536-
xmlBody.append("</CSV>");
537-
xmlBody.append("</InputSerialization>");
538-
OutputSerialization outputSerialization = request.getOutputSerialization();
539-
CSVFormat csvOutputFormat = outputSerialization.getCsvOutputFormat();
540-
xmlBody.append("<OutputSerialization>");
541-
xmlBody.append("<CSV>");
542-
xmlBody.append("<RecordDelimiter>" + BinaryUtil.toBase64String(csvOutputFormat.getRecordDelimiter().getBytes()) + "</RecordDelimiter>");
543-
xmlBody.append("<FieldDelimiter>" + BinaryUtil.toBase64String(csvOutputFormat.getFieldDelimiter().toString().getBytes()) + "</FieldDelimiter>");
544-
xmlBody.append("<QuoteCharacter>" + BinaryUtil.toBase64String(csvOutputFormat.getQuoteChar().toString().getBytes()) + "</QuoteCharacter>");
545-
xmlBody.append("</CSV>");
546-
xmlBody.append("<KeepAllColumns>" + outputSerialization.isKeepAllColumns() + "</KeepAllColumns>");
547-
xmlBody.append("<OutputRawData>" + outputSerialization.isOutputRawData() + "</OutputRawData>");
548-
xmlBody.append("<OutputHeader>" + outputSerialization.isOutputHeader() + "</OutputHeader>");
549-
xmlBody.append("<EnablePayloadCrc>" + outputSerialization.isPayloadCrcEnabled() + "</EnablePayloadCrc>");
550-
xmlBody.append("</OutputSerialization>");
596+
SelectContentFormat selectContentFormat = inputSerialization.getSelectContentFormat();
597+
598+
if (selectContentFormat == SelectContentFormat.JSON) {
599+
populateSelectJsonObjectRequest(xmlBody, request);
600+
} else {
601+
populateSelectCsvObjectRequest(xmlBody, request);
602+
}
603+
551604
xmlBody.append("</SelectRequest>");
552605

553606
try {

‎src/main/java/com/aliyun/oss/internal/OSSObjectOperation.java‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,6 @@
3131
import static com.aliyun.oss.event.ProgressPublisher.publishProgress;
3232
import static com.aliyun.oss.internal.OSSConstants.DEFAULT_BUFFER_SIZE;
3333
import static com.aliyun.oss.internal.OSSConstants.DEFAULT_CHARSET_NAME;
34-
import static com.aliyun.oss.internal.OSSHeaders.OSS_SELECT_CSV_ROWS;
35-
import static com.aliyun.oss.internal.OSSHeaders.OSS_SELECT_CSV_SPLITS;
3634
import static com.aliyun.oss.internal.OSSHeaders.OSS_SELECT_OUTPUT_RAW;
3735
import static com.aliyun.oss.internal.OSSUtils.OSS_RESOURCE_MANAGER;
3836
import static com.aliyun.oss.internal.OSSUtils.addDateHeader;
@@ -225,8 +223,17 @@ public SelectObjectMetadata createSelectObjectMetadata(CreateSelectObjectMetadat
225223
OSSObject ossObject = doOperation(request, new GetObjectResponseParser(bucketName, key), bucketName, key, true);
226224
publishProgress(selectProgressListener, ProgressEventType.SELECT_STARTED_EVENT);
227225
SelectObjectMetadata selectObjectMetadata = new SelectObjectMetadata(ossObject.getObjectMetadata());
226+
SelectObjectMetadata.SelectContentMetadataBase selectContentMetadataBase;
227+
if (createSelectObjectMetadataRequest.getInputSerialization().getSelectContentFormat() == SelectContentFormat.CSV) {
228+
selectObjectMetadata.setCsvObjectMetadata(new SelectObjectMetadata.CSVObjectMetadata());
229+
selectContentMetadataBase = selectObjectMetadata.getCsvObjectMetadata();
230+
} else {
231+
selectObjectMetadata.setJsonObjectMetadata(new SelectObjectMetadata.JsonObjectMetadata());
232+
selectContentMetadataBase = selectObjectMetadata.getJsonObjectMetadata();
233+
}
228234
InputStream in = ossObject.getObjectContent();
229-
CreateSelectMetaInputStream warppedStream = new CreateSelectMetaInputStream(in, selectObjectMetadata, selectProgressListener);
235+
CreateSelectMetaInputStream warppedStream = new CreateSelectMetaInputStream(in, selectContentMetadataBase, selectProgressListener);
236+
warppedStream.setRequestId(ossObject.getRequestId());
230237
while (warppedStream.read() != -1) {
231238
//read until eof
232239
}
@@ -286,8 +293,10 @@ public OSSObject selectObject(SelectObjectRequest selectObjectRequest) throws OS
286293
publishProgress(selectProgressListener, ProgressEventType.SELECT_STARTED_EVENT);
287294
InputStream inputStream = ossObject.getObjectContent();
288295
if (!Boolean.parseBoolean(ossObject.getObjectMetadata().getRawMetadata().get(OSS_SELECT_OUTPUT_RAW).toString())) {
289-
ossObject.setObjectContent(new SelectInputStream(inputStream, selectProgressListener,
290-
selectObjectRequest.getOutputSerialization().isPayloadCrcEnabled()));
296+
SelectInputStream selectInputStream = new SelectInputStream(inputStream, selectProgressListener,
297+
selectObjectRequest.getOutputSerialization().isPayloadCrcEnabled());
298+
selectInputStream.setRequestId(ossObject.getRequestId());
299+
ossObject.setObjectContent(selectInputStream);
291300
}
292301
return ossObject;
293302
} catch (RuntimeException e) {

‎src/main/java/com/aliyun/oss/internal/RequestParameters.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ public final class RequestParameters {
5151
public static final String SUBRESOURCE_PROCESS = "x-oss-process";
5252
public static final String SUBRESOURCE_CSV_SELECT = "csv/select";
5353
public static final String SUBRESOURCE_CSV_META = "csv/meta";
54+
public static final String SUBRESOURCE_JSON_SELECT = "json/select";
55+
public static final String SUBRESOURCE_JSON_META = "json/meta";
5456
public static final String SUBRESOURCE_SQL = "sql";
5557
public static final String SUBRESOURCE_SYMLINK = "symlink";
5658
public static final String SUBRESOURCE_STAT = "stat";

0 commit comments

Comments
 (0)