Merge pull request #427 from zezha-msft/1.1.0

Version 1.1.0: July 2017 service release

Author: zezha-msft

azure-storage-blob/ChangeLog.md

@@ -2,6 +2,12 @@
 
 > See [BreakingChanges](BreakingChanges.md) for a detailed list of API breaks.
 
+## Version 1.1.0:
+
+- Support for 2017-07-29 REST version. Please see our REST API documentation and blogs for information about the related added features.
+- Added support for soft delete feature. If a delete retention policy is enabled through the set service properties API, then blobs or snapshots could be deleted softly and retained for a specified number of days, before being permanently removed by garbage collection.
+- Error message now contains the ErrorCode from the x-ms-error-code header value.
+
 ## Version 1.0.0:
 
 - The package has switched from Apache 2.0 to the MIT license.
@@ -14,4 +20,4 @@
 
 - Enabling MD5 validation no longer uses the memory-efficient algorithm for large block blobs, since computing the MD5 hash requires reading the entire block into memory.
 - Fixed a bug in the _SubStream class which was at risk of causing data corruption when using the memory-efficient algorithm for large block blobs.
-- Support for AccessTierChangeTime to get the last time a tier was modified on an individual blob.
+- Support for AccessTierChangeTime to get the last time a tier was modified on an individual blob.

azure-storage-blob/azure/storage/blob/_constants.py

@@ -5,10 +5,10 @@
 # --------------------------------------------------------------------------
 
 __author__ = 'Microsoft Corp. <[email protected]>'
-__version__ = '1.0.0'
+__version__ = '1.1.0'
 
 # x-ms-version for storage service.
-X_MS_VERSION = '2017-04-17'
+X_MS_VERSION = '2017-07-29'
 
 # internal configurations, should not be changed
 _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024

azure-storage-blob/azure/storage/blob/_deserialization.py

@@ -17,7 +17,7 @@
 )
 from azure.storage.common._deserialization import (
     _parse_properties,
-    _int_to_str,
+    _to_int,
     _parse_metadata,
     _convert_xml_to_signed_identifiers,
     _bool,
@@ -62,7 +62,7 @@ def _parse_page_properties(response):
     put_page = PageBlobProperties()
     put_page.last_modified = parser.parse(response.headers.get('last-modified'))
     put_page.etag = response.headers.get('etag')
-    put_page.sequence_number = _int_to_str(response.headers.get('x-ms-blob-sequence-number'))
+    put_page.sequence_number = _to_int(response.headers.get('x-ms-blob-sequence-number'))
 
     return put_page
 
@@ -74,8 +74,8 @@ def _parse_append_block(response):
     append_block = AppendBlockProperties()
     append_block.last_modified = parser.parse(response.headers.get('last-modified'))
     append_block.etag = response.headers.get('etag')
-    append_block.append_offset = _int_to_str(response.headers.get('x-ms-blob-append-offset'))
-    append_block.committed_block_count = _int_to_str(response.headers.get('x-ms-blob-committed-block-count'))
+    append_block.append_offset = _to_int(response.headers.get('x-ms-blob-append-offset'))
+    append_block.committed_block_count = _to_int(response.headers.get('x-ms-blob-committed-block-count'))
 
     return append_block
 
@@ -95,7 +95,7 @@ def _parse_lease(response):
     '''
     lease = {'time': response.headers.get('x-ms-lease-time')}
     if lease['time']:
-        lease['time'] = _int_to_str(lease['time'])
+        lease['time'] = _to_int(lease['time'])
 
     lease['id'] = response.headers.get('x-ms-lease-id')
 
@@ -215,9 +215,9 @@ def _convert_xml_to_containers(response):
 LIST_BLOBS_ATTRIBUTE_MAP = {
     'Last-Modified': (None, 'last_modified', parser.parse),
     'Etag': (None, 'etag', _to_str),
-    'x-ms-blob-sequence-number': (None, 'sequence_number', _int_to_str),
+    'x-ms-blob-sequence-number': (None, 'sequence_number', _to_int),
     'BlobType': (None, 'blob_type', _to_str),
-    'Content-Length': (None, 'content_length', _int_to_str),
+    'Content-Length': (None, 'content_length', _to_int),
     'ServerEncrypted': (None, 'server_encrypted', _bool),
     'Content-Type': ('content_settings', 'content_type', _to_str),
     'Content-Encoding': ('content_settings', 'content_encoding', _to_str),
@@ -238,6 +238,8 @@ def _convert_xml_to_containers(response):
     'AccessTierChangeTime': (None, 'blob_tier_change_time', parser.parse),
     'AccessTierInferred': (None, 'blob_tier_inferred', _bool),
     'ArchiveStatus': (None, 'rehydration_status', _to_str),
+    'DeletedTime': (None, 'deleted_time', parser.parse),
+    'RemainingRetentionDays': (None, 'remaining_retention_days', _to_int),
 }
 
 
@@ -252,6 +254,7 @@ def _convert_xml_to_blob_list(response):
       <Blobs>
         <Blob>
           <Name>blob-name</name>
+          <Deleted>true</Deleted>
           <Snapshot>date-time-value</Snapshot>
           <Properties>
             <Last-Modified>date-time-value</Last-Modified>
@@ -276,6 +279,8 @@ def _convert_xml_to_blob_list(response):
             <AccessTier>P4 | P6 | P10 | P20 | P30 | P40 | P50 | P60 | Archive | Cool | Hot</AccessTier>
             <AccessTierChangeTime>date-time-value</AccessTierChangeTime>
             <AccessTierInferred>true</AccessTierInferred>
+            <DeletedTime>datetime</DeletedTime>
+            <RemainingRetentionDays>int</RemainingRetentionDays>
           </Properties>
           <Metadata>   
             <Name>value</Name>
@@ -309,6 +314,10 @@ def _convert_xml_to_blob_list(response):
         blob.name = blob_element.findtext('Name')
         blob.snapshot = blob_element.findtext('Snapshot')
 
+        deleted = blob_element.findtext('Deleted')
+        if deleted:
+            blob.deleted = _bool(deleted)
+
         # Properties
         properties_element = blob_element.find('Properties')
         if properties_element is not None:

azure-storage-blob/azure/storage/blob/appendblobservice.py

@@ -120,7 +120,7 @@ def create_blob(self, container_name, blob_name, content_settings=None,
                     if_modified_since=None, if_unmodified_since=None,
                     if_match=None, if_none_match=None, timeout=None):
         '''
-        Creates a blob or overrides an existing blob. Use if_match=* to
+        Creates a blob or overrides an existing blob. Use if_none_match=* to
         prevent overriding an existing blob. 
 
         See create_blob_from_* for high level

azure-storage-blob/azure/storage/blob/baseblobservice.py

@@ -1278,6 +1278,10 @@ def _list_blobs(self, container_name, prefix=None, marker=None,
                     Version 2012-02-12 and newer. Specifies that metadata
                     related to any current or previous Copy Blob operation
                     should be included in the response.
+                deleted:
+                    Version 2017-07-29 and newer. Specifies that soft deleted blobs
+                    which are retained by the service should be included
+                    in the response.
         :param str delimiter:
             When the request includes this parameter, the operation
             returns a :class:`~azure.storage.blob.models.BlobPrefix` element in the response body that acts as a
@@ -1343,30 +1347,41 @@ def get_blob_service_stats(self, timeout=None):
 
     def set_blob_service_properties(
             self, logging=None, hour_metrics=None, minute_metrics=None,
-            cors=None, target_version=None, timeout=None):
+            cors=None, target_version=None, timeout=None, delete_retention_policy=None):
         '''
         Sets the properties of a storage account's Blob service, including
         Azure Storage Analytics. If an element (ex Logging) is left as None, the 
         existing settings on the service for that functionality are preserved.
 
-        :param Logging logging:
+        :param logging:
             Groups the Azure Analytics Logging settings.
-        :param Metrics hour_metrics:
+        :type logging:
+            :class:`~azure.storage.common.models.Logging`
+        :param hour_metrics:
             The hour metrics settings provide a summary of request 
             statistics grouped by API in hourly aggregates for blobs.
-        :param Metrics minute_metrics:
+        :type hour_metrics:
+            :class:`~azure.storage.common.models.Metrics`
+        :param minute_metrics:
             The minute metrics settings provide request statistics 
             for each minute for blobs.
+        :type minute_metrics:
+            :class:`~azure.storage.common.models.Metrics`
         :param cors:
             You can include up to five CorsRule elements in the 
             list. If an empty list is specified, all CORS rules will be deleted, 
             and CORS will be disabled for the service.
         :type cors: list(:class:`~azure.storage.common.models.CorsRule`)
-        :param string target_version:
+        :param str target_version:
             Indicates the default version to use for requests if an incoming 
             request's version is not specified. 
         :param int timeout:
             The timeout parameter is expressed in seconds.
+        :param delete_retention_policy:
+            The delete retention policy specifies whether to retain deleted blobs.
+            It also specifies the number of days and versions of blob to keep.
+        :type delete_retention_policy:
+            :class:`~azure.storage.common.models.DeleteRetentionPolicy`
         '''
         request = HTTPRequest()
         request.method = 'PUT'
@@ -1378,7 +1393,8 @@ def set_blob_service_properties(
             'timeout': _int_to_str(timeout),
         }
         request.body = _get_request_body(
-            _convert_service_properties_to_xml(logging, hour_metrics, minute_metrics, cors, target_version))
+            _convert_service_properties_to_xml(logging, hour_metrics, minute_metrics,
+                                               cors, target_version, delete_retention_policy))
 
         self._perform_request(request)
 
@@ -1389,10 +1405,8 @@ def get_blob_service_properties(self, timeout=None):
 
         :param int timeout:
             The timeout parameter is expressed in seconds.
-        :return: The blob service properties.
-        :rtype:
-            :class:`~azure.storage.common.models.ServiceProperties` with an attached
-            target_version property
+        :return: The blob :class:`~azure.storage.common.models.ServiceProperties` with an attached
+            target_version property.
         '''
         request = HTTPRequest()
         request.method = 'GET'
@@ -3136,6 +3150,12 @@ def delete_blob(self, container_name, blob_name, snapshot=None,
         snapshots. You can delete both at the same time with the Delete
         Blob operation.
 
+        If a delete retention policy is enabled for the service, then this operation soft deletes the blob or snapshot
+        and retains the blob or snapshot for specified number of days.
+        After specified number of days, blob's data is removed from the service during garbage collection.
+        Soft deleted blob or snapshot is accessible through List Blobs API specifying include=Include.Deleted option.
+        Soft-deleted blob or snapshot can be restored using Undelete API.
+
         :param str container_name:
             Name of existing container.
         :param str blob_name:
@@ -3191,3 +3211,28 @@ def delete_blob(self, container_name, blob_name, snapshot=None,
         }
 
         self._perform_request(request)
+
+    def undelete_blob(self, container_name, blob_name, timeout=None):
+        '''
+        The undelete Blob operation restores the contents and metadata of soft deleted blob or snapshot.
+        Attempting to undelete a blob or snapshot that is not soft deleted will succeed without any changes.
+
+        :param str container_name:
+            Name of existing container.
+        :param str blob_name:
+            Name of existing blob.
+        :param int timeout:
+            The timeout parameter is expressed in seconds.
+        '''
+        _validate_not_none('container_name', container_name)
+        _validate_not_none('blob_name', blob_name)
+        request = HTTPRequest()
+        request.method = 'PUT'
+        request.host_locations = self._get_host_locations()
+        request.path = _get_path(container_name, blob_name)
+        request.query = {
+            'comp': 'undelete',
+            'timeout': _int_to_str(timeout)
+        }
+
+        self._perform_request(request)

azure-storage-blob/azure/storage/blob/models.py

@@ -65,14 +65,20 @@ class Blob(object):
         Stores all the system properties for the blob.
     :ivar metadata:
         Name-value pairs associated with the blob as metadata.
+    :ivar bool deleted:
+        Specify whether the blob was soft deleted.
+        In other words, if the blob is being retained by the delete retention policy,
+        this field would be True. The blob could be undeleted or it will be garbage collected after the specified
+        time period.
     '''
 
-    def __init__(self, name=None, snapshot=None, content=None, props=None, metadata=None):
+    def __init__(self, name=None, snapshot=None, content=None, props=None, metadata=None, deleted=False):
         self.name = name
         self.snapshot = snapshot
         self.content = content
         self.properties = props or BlobProperties()
         self.metadata = metadata
+        self.deleted = deleted
 
 
 class BlobProperties(object):
@@ -118,6 +124,10 @@ class BlobProperties(object):
     :ivar bool blob_tier_inferred:
         Indicates whether the access tier was inferred by the service.
         If false, it indicates that the tier was set explicitly.
+    :ivar datetime deleted_time:
+        A datetime object representing the time at which the blob was deleted.
+    :ivar int remaining_retention_days:
+        The number of days that the blob will be retained before being permanently deleted by the service.
     '''
 
     def __init__(self):
@@ -135,6 +145,8 @@ def __init__(self):
         self.blob_tier = None
         self.blob_tier_change_time = None
         self.blob_tier_inferred = False
+        self.deleted_time = None
+        self.remaining_retention_days = None
 
 
 class ContentSettings(object):
@@ -506,10 +518,12 @@ class Include(object):
     :ivar ~azure.storage.blob.models.Include Include.UNCOMMITTED_BLOBS: 
         Specifies that blobs for which blocks have been uploaded, but which have not 
         been committed using Put Block List, be included in the response.
+    :ivar ~azure.storage.blob.models.Include Include.DELETED:
+        Specifies that deleted blobs should be returned in the response.
     '''
 
     def __init__(self, snapshots=False, metadata=False, uncommitted_blobs=False,
-                 copy=False, _str=None):
+                 copy=False, deleted=False, _str=None):
         '''
         :param bool snapshots:
              Specifies that snapshots should be included in the enumeration.
@@ -520,7 +534,9 @@ def __init__(self, snapshots=False, metadata=False, uncommitted_blobs=False,
             not been committed using Put Block List, be included in the response.
         :param bool copy: 
             Specifies that metadata related to any current or previous Copy Blob 
-            operation should be included in the response. 
+            operation should be included in the response.
+        :param bool deleted:
+            Specifies that deleted blobs should be returned in the response.
         :param str _str: 
             A string representing the includes.
         '''
@@ -531,6 +547,7 @@ def __init__(self, snapshots=False, metadata=False, uncommitted_blobs=False,
         self.metadata = metadata or ('metadata' in components)
         self.uncommitted_blobs = uncommitted_blobs or ('uncommittedblobs' in components)
         self.copy = copy or ('copy' in components)
+        self.deleted = deleted or ('deleted' in components)
 
     def __or__(self, other):
         return Include(_str=str(self) + str(other))
@@ -542,14 +559,16 @@ def __str__(self):
         include = (('snapshots,' if self.snapshots else '') +
                    ('metadata,' if self.metadata else '') +
                    ('uncommittedblobs,' if self.uncommitted_blobs else '') +
-                   ('copy,' if self.copy else ''))
+                   ('copy,' if self.copy else '') +
+                   ('deleted,' if self.deleted else ''))
         return include.rstrip(',')
 
 
 Include.COPY = Include(copy=True)
 Include.METADATA = Include(metadata=True)
 Include.SNAPSHOTS = Include(snapshots=True)
 Include.UNCOMMITTED_BLOBS = Include(uncommitted_blobs=True)
+Include.DELETED = Include(deleted=True)
 
 
 class BlobPermissions(object):

azure-storage-blob/setup.py

@@ -51,7 +51,7 @@
 
 setup(
     name='azure-storage-blob',
-    version='1.0.0',
+    version='1.1.0',
     description='Microsoft Azure Storage Blob Client Library for Python',
     long_description=open('README.rst', 'r').read(),
     license='MIT License',
@@ -74,7 +74,7 @@
     packages=find_packages(),
     install_requires=[
                          'azure-common>=1.1.5',
-                         'azure-storage-common>=1.0.0,<1.1.0'
+                         'azure-storage-common>=1.1.0,<1.2.0'
                      ],
     extras_require={
         ":python_version<'3.0'": ['futures'],

azure-storage-common/BreakingChanges.md

@@ -2,6 +2,10 @@
 
 > See the [Change Log](ChangeLog.md) for a summary of storage library changes.
 
+## Version 1.1.0:
+
+- Error message now contains the ErrorCode from the x-ms-error-code header value.
+
 ## Version 1.0.0:
 
 - Renamed the confusing argument name increment_power to increment_base on ExponentialRetry.

azure-storage-common/ChangeLog.md

@@ -2,6 +2,11 @@
 
 > See [BreakingChanges](BreakingChanges.md) for a detailed list of API breaks.
 
+## Version 1.1.0:
+
+- Support for 2017-07-29 REST version. Please see our REST API documentation and blogs for information about the related added features.
+- Error message now contains the ErrorCode from the x-ms-error-code header value.
+
 ## Version 1.0.0:
 
 - The package has switched from Apache 2.0 to the MIT license.

azure-storage-common/azure/storage/common/__init__.py

@@ -14,6 +14,7 @@
     Logging,
     Metrics,
     CorsRule,
+    DeleteRetentionPolicy,
     ServiceProperties,
     AccessPolicy,
     ResourceTypes,