Merge pull request #624 from Azure/dev

2.1.0 Release for Blob/File/Queue/Common

Author: zezha-msft

azure-storage-blob/ChangeLog.md

@@ -2,7 +2,16 @@
 
 > See [BreakingChanges](BreakingChanges.md) for a detailed list of API breaks.
 
+## Version 2.1.0:
+
+- Support for 2019-02-02 REST version. Please see our REST API documentation and blog for information about the related added features.
+- Added Batch Delete Blob API.
+- Added Batch Set Standard Blob Tier API(for BlockBlob).
+- Added Blob Tier support for PutBlob/PutBlockList/CopyBlob APIs.
+- Added support for client provided encryption key to numerous APIs. 
+
 ## Version 2.0.1:
+
 - Updated dependency on azure-storage-common.
 
 ## Version 2.0.0:

azure-storage-blob/azure/storage/blob/__init__.py

@@ -27,5 +27,11 @@
     PublicAccess,
     BlobPrefix,
     DeleteSnapshot,
+    BatchDeleteSubRequest,
+    BatchSetBlobTierSubRequest,
+    BatchSubResponse,
+    CustomerProvidedEncryptionKey,
+    RehydratePriority,
 )
 from .pageblobservice import PageBlobService
+from ._constants import __version__

azure-storage-blob/azure/storage/blob/_constants.py

@@ -5,10 +5,10 @@
 # --------------------------------------------------------------------------
 
 __author__ = 'Microsoft Corp. <[email protected]>'
-__version__ = '2.0.1'
+__version__ = '2.1.0'
 
 # x-ms-version for storage service.
-X_MS_VERSION = '2018-11-09'
+X_MS_VERSION = '2019-02-02'
 
 # internal configurations, should not be changed
 _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024

azure-storage-blob/azure/storage/blob/_deserialization.py

@@ -6,6 +6,8 @@
 from azure.common import AzureException
 from dateutil import parser
 
+from azure.storage.common._http import HTTPResponse
+
 try:
     from xml.etree import cElementTree as ETree
 except ImportError:
@@ -36,14 +38,23 @@
     ResourceProperties,
     BlobPrefix,
     AccountInformation,
-    UserDelegationKey,
-)
+    UserDelegationKey, BatchSubResponse)
 from ._encryption import _decrypt_blob
 from azure.storage.common.models import _list
 from azure.storage.common._error import (
     _validate_content_match,
     _ERROR_DECRYPTION_FAILURE,
 )
+from io import BytesIO
+
+_HTTP_LINE_ENDING = "\r\n"
+
+def _parse_cpk_headers(response, properties):
+    server_encrypted = response.headers.get('x-ms-request-server-encrypted')
+    if server_encrypted is not None:
+        properties.request_server_encrypted = _bool(server_encrypted)
+
+    properties.encryption_key_sha256 = response.headers.get('x-ms-encryption-key-sha256')
 
 
 def _parse_base_properties(response):
@@ -53,6 +64,7 @@ def _parse_base_properties(response):
     resource_properties = ResourceProperties()
     resource_properties.last_modified = parser.parse(response.headers.get('last-modified'))
     resource_properties.etag = response.headers.get('etag')
+    _parse_cpk_headers(response, resource_properties)
 
     return resource_properties
 
@@ -65,6 +77,7 @@ def _parse_page_properties(response):
     put_page.last_modified = parser.parse(response.headers.get('last-modified'))
     put_page.etag = response.headers.get('etag')
     put_page.sequence_number = _to_int(response.headers.get('x-ms-blob-sequence-number'))
+    _parse_cpk_headers(response, put_page)
 
     return put_page
 
@@ -78,6 +91,7 @@ def _parse_append_block(response):
     append_block.etag = response.headers.get('etag')
     append_block.append_offset = _to_int(response.headers.get('x-ms-blob-append-offset'))
     append_block.committed_block_count = _to_int(response.headers.get('x-ms-blob-committed-block-count'))
+    _parse_cpk_headers(response, append_block)
 
     return append_block
 
@@ -290,7 +304,7 @@ def _convert_xml_to_blob_list(response):
             <RemainingRetentionDays>int</RemainingRetentionDays>
             <Creation-Time>date-time-value</Creation-Time>
           </Properties>
-          <Metadata>   
+          <Metadata>
             <Name>value</Name>
           </Metadata>
         </Blob>
@@ -392,7 +406,7 @@ def _convert_xml_to_blob_name_list(response):
             <RemainingRetentionDays>int</RemainingRetentionDays>
             <Creation-Time>date-time-value</Creation-Time>
           </Properties>
-          <Metadata>   
+          <Metadata>
             <Name>value</Name>
           </Metadata>
         </Blob>
@@ -475,19 +489,19 @@ def _convert_xml_to_page_ranges(response):
     '''
     <?xml version="1.0" encoding="utf-8"?>
     <PageList>
-       <PageRange> 
-          <Start>Start Byte</Start> 
-          <End>End Byte</End> 
-       </PageRange> 
-       <ClearRange> 
-          <Start>Start Byte</Start> 
-          <End>End Byte</End> 
-       </ClearRange> 
-       <PageRange> 
-          <Start>Start Byte</Start> 
-          <End>End Byte</End> 
-       </PageRange> 
-    </PageList> 
+       <PageRange>
+          <Start>Start Byte</Start>
+          <End>End Byte</End>
+       </PageRange>
+       <ClearRange>
+          <Start>Start Byte</Start>
+          <End>End Byte</End>
+       </ClearRange>
+       <PageRange>
+          <Start>Start Byte</Start>
+          <End>End Byte</End>
+       </PageRange>
+    </PageList>
     '''
     if response is None or response.body is None:
         return None
@@ -554,3 +568,83 @@ def _convert_xml_to_user_delegation_key(response):
     delegation_key.value = key_element.findtext('Value')
 
     return delegation_key
+
+
+def _ingest_batch_response(batch_response, batch_sub_requests):
+    """
+    Takes the response to a batch request and parses the response into the separate responses.
+
+    :param :class:`~azure.storage.common._http.HTTPResponse` batch_response:
+        batchResponse The response of the HTTP batch request generated by this object.
+    :return: sub-responses parsed from batch HTTP response
+    :rtype: list of :class:`~azure.storage.common._http.HTTPResponse`
+    """
+    parsed_batch_sub_response_list = []
+
+    # header value format: `multipart/mixed; boundary=<delimiter>`
+    response_delimiter = batch_response.headers.get('content-type').split("=")[1]
+
+    response_body = batch_response.body.decode('utf-8')
+
+    # split byte[] on the "substring" "--<delim>\r\n"
+    sub_response_list = response_body.split("--" + response_delimiter + _HTTP_LINE_ENDING)
+
+    # strip final, slightly different delim "\r\n--<delim>--" off last entry
+    sub_response_list[len(sub_response_list) - 1] = \
+        sub_response_list[len(sub_response_list) - 1].split(_HTTP_LINE_ENDING + "--" + response_delimiter + "--")[0]
+
+    for sub_response in sub_response_list:
+        if len(sub_response) != 0:
+            http_response = _parse_sub_response_to_http_response(sub_response)
+            is_successful = 200 <= http_response.status < 300
+            index_of_sub_request = _to_int(http_response.headers.get('Content-ID'))
+            batch_sub_request = batch_sub_requests[index_of_sub_request]
+
+            parsed_batch_sub_response_list.append(BatchSubResponse(is_successful, http_response, batch_sub_request))
+
+    return parsed_batch_sub_response_list
+
+
+def _parse_sub_response_to_http_response(sub_response):
+    """
+    Header: Value (1 or more times)
+
+    HTTP/<version> <statusCode> <statusName>
+    Header: Value (1 or more times)
+
+    body (if any)
+
+    :param sub_response:
+        The raw bytes of this sub-response.
+    :return: An HttpResponse object.
+    """
+
+    empty_line = _HTTP_LINE_ENDING.encode('utf-8')
+    num_empty_lines = 0
+    batch_http_sub_response = HTTPResponse(None, '', dict(), b'')
+    try:
+        body_stream = BytesIO()
+        body_stream.write(sub_response.encode('utf-8'))
+        body_stream.seek(0)
+
+        while True:
+            line = body_stream.readline()
+            if line == b'':
+                return batch_http_sub_response
+
+            if line.startswith("HTTP".encode('utf-8')):
+                batch_http_sub_response.status = _to_int(line.decode('utf-8').split(" ")[1])
+            elif line == empty_line:
+                num_empty_lines += 1
+            elif line.startswith("x-ms-error-code".encode('utf-8')):
+                batch_http_sub_response.message = line.decode('utf-8').split(": ")[1].rstrip()
+            elif num_empty_lines is 2:
+                batch_http_sub_response.body += line
+            else:
+                header = line.decode('utf-8').split(": ")[0]
+                value = line.decode('utf-8').split(": ")[1].rstrip()
+                batch_http_sub_response.headers[header] = value
+    finally:
+        body_stream.close()
+
+    return batch_http_sub_response

azure-storage-blob/azure/storage/blob/_download_chunking.py

@@ -10,7 +10,7 @@ def _download_blob_chunks(blob_service, container_name, blob_name, snapshot,
                           download_size, block_size, progress, start_range, end_range,
                           stream, max_connections, progress_callback, validate_content,
                           lease_id, if_modified_since, if_unmodified_since, if_match,
-                          if_none_match, timeout, operation_context):
+                          if_none_match, timeout, operation_context, cpk):
 
     downloader_class = _ParallelBlobChunkDownloader if max_connections > 1 else _SequentialBlobChunkDownloader
 
@@ -34,6 +34,7 @@ def _download_blob_chunks(blob_service, container_name, blob_name, snapshot,
         if_none_match,
         timeout,
         operation_context,
+        cpk,
     )
 
     if max_connections > 1:
@@ -49,7 +50,7 @@ class _BlobChunkDownloader(object):
     def __init__(self, blob_service, container_name, blob_name, snapshot, download_size,
                  chunk_size, progress, start_range, end_range, stream,
                  progress_callback, validate_content, lease_id, if_modified_since,
-                 if_unmodified_since, if_match, if_none_match, timeout, operation_context):
+                 if_unmodified_since, if_match, if_none_match, timeout, operation_context, cpk):
         # identifiers for the blob
         self.blob_service = blob_service
         self.container_name = container_name
@@ -78,6 +79,7 @@ def __init__(self, blob_service, container_name, blob_name, snapshot, download_s
         self.if_unmodified_since = if_unmodified_since
         self.if_match = if_match
         self.if_none_match = if_none_match
+        self.cpk = cpk
 
     def get_chunk_offsets(self):
         index = self.start_index
@@ -119,7 +121,8 @@ def _download_chunk(self, chunk_start, chunk_end):
             if_match=self.if_match,
             if_none_match=self.if_none_match,
             timeout=self.timeout,
-            _context=self.operation_context
+            _context=self.operation_context,
+            cpk=self.cpk,
         )
 
         # This makes sure that if_match is set so that we can validate 
@@ -132,15 +135,15 @@ class _ParallelBlobChunkDownloader(_BlobChunkDownloader):
     def __init__(self, blob_service, container_name, blob_name, snapshot, download_size,
                  chunk_size, progress, start_range, end_range, stream,
                  progress_callback, validate_content, lease_id, if_modified_since,
-                 if_unmodified_since, if_match, if_none_match, timeout, operation_context):
+                 if_unmodified_since, if_match, if_none_match, timeout, operation_context, cpk):
 
         super(_ParallelBlobChunkDownloader, self).__init__(blob_service, container_name, blob_name, snapshot,
                                                            download_size,
                                                            chunk_size, progress, start_range, end_range, stream,
                                                            progress_callback, validate_content, lease_id,
                                                            if_modified_since,
                                                            if_unmodified_since, if_match, if_none_match, timeout,
-                                                           operation_context)
+                                                           operation_context, cpk)
 
         # for a parallel download, the stream is always seekable, so we note down the current position
         # in order to seek to the right place when out-of-order chunks come in