Merge pull request #1340 from CatalystCode/azure-blobs-api-version-update
Update Azure Blob Storage API version to 2016-05-31
diff --git a/CHANGES.rst b/CHANGES.rst
index 3f0ae24..1eccfc1 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -187,11 +187,16 @@
[Clemens Wolff - @c-w]
- [Azure Blobs] Fix a bug with Azure storage driver works when used against a
- storage account that was created using ``kind=BlobStrage``. The includes
- updating minimum API version used / supported by storage driver from
- ``2012-02-12`` to ``2014-02-14'``. (LIBCLOUD-851, GITHUB-1202, GITHUB-1294)
+ storage account that was created using ``kind=BlobStrage``. This includes
+ updating the minimum API version used / supported by the storage driver from
+ ``2012-02-12`` to ``2014-02-14``. (LIBCLOUD-851, GITHUB-1202, GITHUB-1294)
[Clemens Wolff - @c-w, Davis Kirkendall - @daviskirk]
+- [Azure Blobs] Increase the maximum size of block blobs that can be created
+ to 100 MB. This includes updating the minimum API version used / supported
+ by the storage driver from ``2014-02-14`` to ``2016-05-31``. (GITHUB-1340)
+ [Clemens Wolff - @c-w]
+
- [Azure Blobs] Set the minimum required version of requests to ``2.5.0`` since
requests ``2.4.0`` and earlier exhibit XML parsing errors of Azure Storage
responses. (GITHUB-1325, GITHUB-1322)
diff --git a/libcloud/common/azure.py b/libcloud/common/azure.py
index ff81acf..b4bfb9c 100644
--- a/libcloud/common/azure.py
+++ b/libcloud/common/azure.py
@@ -159,22 +159,8 @@
CanonicalizedHeaders +
CanonicalizedResource;
"""
- special_header_values = []
xms_header_values = []
param_list = []
- special_header_keys = [
- 'content-encoding',
- 'content-language',
- 'content-length',
- 'content-md5',
- 'content-type',
- 'date',
- 'if-modified-since',
- 'if-match',
- 'if-none-match',
- 'if-unmodified-since',
- 'range'
- ]
# Split the x-ms headers and normal headers and make everything
# lower case
@@ -188,16 +174,8 @@
headers_copy[header] = value
# Get the values for the headers in the specific order
- for header in special_header_keys:
- header = header.lower() # Just for safety
- if header in headers_copy:
- special_header_values.append(headers_copy[header])
- elif header == "content-length" and method not in ("GET", "HEAD"):
- # Must be '0' unless method is GET or HEAD
- # https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
- special_header_values.append('0')
- else:
- special_header_values.append('')
+ special_header_values = self._format_special_header_values(
+ headers_copy, method)
# Prepare the first section of the string to be signed
values_to_sign = [method] + special_header_values
@@ -229,6 +207,39 @@
return 'SharedKey %s:%s' % (self.user_id, b64_hmac.decode('utf-8'))
+ def _format_special_header_values(self, headers, method):
+ is_change = method not in ('GET', 'HEAD')
+ is_old_api = self.API_VERSION <= '2014-02-14'
+
+ special_header_keys = [
+ 'content-encoding',
+ 'content-language',
+ 'content-length',
+ 'content-md5',
+ 'content-type',
+ 'date',
+ 'if-modified-since',
+ 'if-match',
+ 'if-none-match',
+ 'if-unmodified-since',
+ 'range'
+ ]
+
+ special_header_values = []
+
+ for header in special_header_keys:
+ header = header.lower() # Just for safety
+ if header in headers:
+ special_header_values.append(headers[header])
+ elif header == 'content-length' and is_change and is_old_api:
+ # For old API versions, the Content-Length header must be '0'
+ # https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key#content-length-header-in-version-2014-02-14-and-earlier
+ special_header_values.append('0')
+ else:
+ special_header_values.append('')
+
+ return special_header_values
+
class AzureBaseDriver(object):
name = "Microsoft Azure Service Management API"
diff --git a/libcloud/storage/drivers/azure_blobs.py b/libcloud/storage/drivers/azure_blobs.py
index b7db039..72898c7 100644
--- a/libcloud/storage/drivers/azure_blobs.py
+++ b/libcloud/storage/drivers/azure_blobs.py
@@ -43,15 +43,8 @@
RESPONSES_PER_REQUEST = 100
# As per the Azure documentation, if the upload file size is less than
-# 64MB, we can upload it in a single request. However, in real life azure
-# servers seem to disconnect randomly after around 5 MB or 200s of upload.
-# So, it is better that for file sizes greater than 4MB, we upload it in
-# chunks.
-# Also, with large sizes, if we use a lease, the lease will timeout after
-# 60 seconds, but the upload might still be in progress. This can be
-# handled in code, but if we use chunked uploads, the lease renewal will
-# happen automatically.
-AZURE_BLOCK_MAX_SIZE = 4 * 1024 * 1024
+# 100MB, we can upload it in a single request.
+AZURE_BLOCK_MAX_SIZE = 100 * 1024 * 1024
# Azure block blocks must be maximum 4MB
# Azure page blobs must be aligned in 512 byte boundaries (4MB fits that)
@@ -184,9 +177,7 @@
return action
- # this is the minimum api version supported by storage accounts of kinds
- # StorageV2, Storage and BlobStorage
- API_VERSION = '2014-02-14'
+ API_VERSION = '2016-05-31'
class AzureBlobsStorageDriver(StorageDriver):
diff --git a/libcloud/test/common/test_azure.py b/libcloud/test/common/test_azure.py
new file mode 100644
index 0000000..a945169
--- /dev/null
+++ b/libcloud/test/common/test_azure.py
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import unittest
+
+from libcloud.common.azure import AzureConnection
+from libcloud.test import LibcloudTestCase
+
+
+class AzureConnectionTestCase(LibcloudTestCase):
+ def setUp(self):
+ self.conn = AzureConnection('user', 'key')
+
+ def test_content_length_is_used_if_set(self):
+ headers = {'content-length': '123'}
+ method = 'PUT'
+
+ values = self.conn._format_special_header_values(headers, method)
+
+ self.assertEqual(values[2], '123')
+
+ def test_content_length_is_blank_if_new_api_version(self):
+ headers = {}
+ method = 'PUT'
+ self.conn.API_VERSION = '2018-11-09'
+
+ values = self.conn._format_special_header_values(headers, method)
+
+ self.assertEqual(values[2], '')
+
+ def test_content_length_is_zero_if_write_and_old_api_version(self):
+ headers = {}
+ method = 'PUT'
+ self.conn.API_VERSION = '2011-08-18'
+
+ values = self.conn._format_special_header_values(headers, method)
+
+ self.assertEqual(values[2], '0')
+
+ def test_content_length_is_blank_if_read_and_old_api_version(self):
+ headers = {}
+ method = 'GET'
+ self.conn.API_VERSION = '2011-08-18'
+
+ values = self.conn._format_special_header_values(headers, method)
+
+ self.assertEqual(values[2], '')
+
+
+if __name__ == '__main__':
+ sys.exit(unittest.main())
diff --git a/libcloud/test/storage/fixtures/azure_blobs/list_objects_1.xml b/libcloud/test/storage/fixtures/azure_blobs/list_objects_1.xml
index ec140ad..9358ae4 100644
--- a/libcloud/test/storage/fixtures/azure_blobs/list_objects_1.xml
+++ b/libcloud/test/storage/fixtures/azure_blobs/list_objects_1.xml
@@ -13,9 +13,11 @@
<Content-Language />
<Content-MD5>1B2M2Y8AsgTpgAmY7PhCfg==</Content-MD5>
<Cache-Control />
+ <Content-Disposition />
<BlobType>BlockBlob</BlobType>
<LeaseStatus>unlocked</LeaseStatus>
<LeaseState>available</LeaseState>
+ <ServerEncrypted>true</ServerEncrypted>
</Properties>
<Metadata>
<meta1>value1</meta1>
@@ -33,9 +35,11 @@
<Content-Language />
<Content-MD5>ttgbNgpWctgMJ0MPORU+LA==</Content-MD5>
<Cache-Control />
+ <Content-Disposition />
<BlobType>BlockBlob</BlobType>
<LeaseStatus>unlocked</LeaseStatus>
<LeaseState>available</LeaseState>
+ <ServerEncrypted>true</ServerEncrypted>
</Properties>
<Metadata>
<meta1>value1</meta1>
diff --git a/libcloud/test/storage/fixtures/azure_blobs/list_objects_2.xml b/libcloud/test/storage/fixtures/azure_blobs/list_objects_2.xml
index d71e044..c3afdd4 100644
--- a/libcloud/test/storage/fixtures/azure_blobs/list_objects_2.xml
+++ b/libcloud/test/storage/fixtures/azure_blobs/list_objects_2.xml
@@ -14,9 +14,11 @@
<Content-Language />
<Content-MD5>ttgbNgpWctgMJ0MPORU+LA==</Content-MD5>
<Cache-Control />
+ <Content-Disposition />
<BlobType>BlockBlob</BlobType>
<LeaseStatus>unlocked</LeaseStatus>
<LeaseState>available</LeaseState>
+ <ServerEncrypted>true</ServerEncrypted>
</Properties>
<Metadata />
</Blob>
@@ -30,9 +32,11 @@
<Content-Encoding /><Content-Language />
<Content-MD5>1B2M2Y8AsgTpgAmY7PhCfg==</Content-MD5>
<Cache-Control />
+ <Content-Disposition />
<BlobType>BlockBlob</BlobType>
<LeaseStatus>unlocked</LeaseStatus>
<LeaseState>available</LeaseState>
+ <ServerEncrypted>true</ServerEncrypted>
</Properties>
<Metadata />
</Blob>
