[Storage] Disable decompression on download when using crc64

sdk/storage/azure-storage-blob/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/storage/azure-storage-blob",
-  "Tag": "python/storage/azure-storage-blob_b09e37b521"
+  "Tag": "python/storage/azure-storage-blob_2d1cd89589"
 }

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py

@@ -678,7 +678,13 @@ def download_blob(
         :keyword validate_content:
             Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
             Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
-            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+
+            .. note:: When using CRC64 validation (including when "auto" resolves to CRC64):
+
+                - The ``ext-checksums`` extra must be installed.
+                - Automatic decompression is not supported. If ``decompress=True`` is explicitly
+                  set, a :class:`ValueError` will be raised. If ``decompress`` is not specified,
+                  it will be set to ``False`` automatically.
         :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword lease:
             Required if the blob has an active lease. If specified, download_blob only

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py

@@ -63,7 +63,7 @@
 from ._shared.response_handlers import return_headers_and_deserialized, return_response_headers
 from ._shared.uploads import IterStreamer
 from ._shared.uploads_async import AsyncIterStreamer
-from ._shared.validation import CV_TYPE_PARSED, parse_validation_option
+from ._shared.validation import CV_TYPE_PARSED, is_crc64_validation, parse_validation_option
 from ._upload_helpers import _any_conditions
 
 if TYPE_CHECKING:
@@ -294,6 +294,13 @@ def _download_blob_options(
             config.user_agent_policy.user_agent, sdk_moniker, encryption_options["version"], kwargs
         )
 
+    # Decompression is not supported with CRC64 content validation
+    if is_crc64_validation(validate_content):
+        decompress = kwargs.get("decompress")
+        if decompress is True:
+            raise ValueError("Decompression is not supported when using CRC64 content validation.")
+        kwargs["decompress"] = False
+
     options = {
         "clients": client,
         "config": config,

sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py

@@ -707,7 +707,13 @@ async def download_blob(
         :keyword validate_content:
             Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
             Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
-            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+
+            .. note:: When using CRC64 validation (including when "auto" resolves to CRC64):
+
+                - The ``ext-checksums`` extra must be installed.
+                - Automatic decompression is not supported. If ``decompress=True`` is explicitly
+                  set, a :class:`ValueError` will be raised. If ``decompress`` is not specified,
+                  it will be set to ``False`` automatically.
         :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword lease:
             Required if the blob has an active lease. If specified, download_blob only

sdk/storage/azure-storage-blob/tests/test_content_validation.py

@@ -645,3 +645,23 @@ def hook_fail_once(response):
         blob.commit_block_list([BlobBlock("1")])
         result = blob.download_blob()
         assert result.read() == content
+
+    @BlobPreparer()
+    @pytest.mark.parametrize("a", ["auto", "crc64"])  # a: validate_content
+    @GenericTestProxyParametrize1()
+    @recorded_by_proxy
+    def test_download_decompress_with_crc64(self, a, **kwargs):
+        storage_account_name = kwargs.pop("storage_account_name")
+
+        self._setup(storage_account_name)
+        blob = self.container.get_blob_client(self._get_blob_reference())
+        data = b"abc" * 512
+        blob.upload_blob(data, overwrite=True)
+
+        # decompress=True should raise ValueError
+        with pytest.raises(ValueError, match="Decompression is not supported when using CRC64 content validation."):
+            blob.download_blob(validate_content=a, decompress=True)
+
+        # decompress=False should work fine
+        downloader = blob.download_blob(validate_content=a, decompress=False)
+        assert downloader.read() == data

sdk/storage/azure-storage-blob/tests/test_content_validation_async.py

@@ -628,3 +628,23 @@ def hook_fail_once(response):
         await blob.commit_block_list([BlobBlock("1")])
         result = await blob.download_blob()
         assert await result.read() == content
+
+    @BlobPreparer()
+    @pytest.mark.parametrize("a", ["auto", "crc64"])  # a: validate_content
+    @GenericTestProxyParametrize1()
+    @recorded_by_proxy_async
+    async def test_download_decompress_with_crc64(self, a, **kwargs):
+        storage_account_name = kwargs.pop("storage_account_name")
+
+        await self._setup(storage_account_name)
+        blob = self.container.get_blob_client(self._get_blob_reference())
+        data = b"abc" * 512
+        await blob.upload_blob(data, overwrite=True)
+
+        # decompress=True should raise ValueError
+        with pytest.raises(ValueError, match="Decompression is not supported when using CRC64 content validation."):
+            await blob.download_blob(validate_content=a, decompress=True)
+
+        # decompress=False should work fine
+        downloader = await blob.download_blob(validate_content=a, decompress=False)
+        assert await downloader.read() == data

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py

@@ -688,7 +688,13 @@ def download_file(
         :keyword validate_content:
             Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
             Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
-            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+
+            .. note:: When using CRC64 validation (including when "auto" resolves to CRC64):
+
+                - The ``ext-checksums`` extra must be installed.
+                - Automatic decompression is not supported. If ``decompress=True`` is explicitly
+                  set, a :class:`ValueError` will be raised. If ``decompress`` is not specified,
+                  it will be set to ``False`` automatically.
         :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword int timeout:
             Sets the server-side timeout for the operation in seconds. For more details see

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.py

@@ -710,7 +710,13 @@ async def download_file(
         :keyword validate_content:
             Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
             Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
-            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+
+            .. note:: When using CRC64 validation (including when "auto" resolves to CRC64):
+
+                - The ``ext-checksums`` extra must be installed.
+                - Automatic decompression is not supported. If ``decompress=True`` is explicitly
+                  set, a :class:`ValueError` will be raised. If ``decompress`` is not specified,
+                  it will be set to ``False`` automatically.
         :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword int timeout:
             Sets the server-side timeout for the operation in seconds. For more details see

sdk/storage/azure-storage-file-share/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/storage/azure-storage-file-share",
-  "Tag": "python/storage/azure-storage-file-share_d5f376c42f"
+  "Tag": "python/storage/azure-storage-file-share_bcf00830c4"
 }

sdk/storage/azure-storage-file-share/azure/storage/fileshare/_file_client.py

@@ -42,7 +42,7 @@
 from ._shared.request_handlers import add_metadata_headers, get_length
 from ._shared.response_handlers import return_response_headers, process_storage_error
 from ._shared.uploads import IterStreamer, FileChunkUploader, upload_data_chunks
-from ._shared.validation import CV_TYPE_PARSED, parse_validation_option
+from ._shared.validation import CV_TYPE_PARSED, is_crc64_validation, parse_validation_option
 
 if TYPE_CHECKING:
     from azure.core.credentials import AzureNamedKeyCredential, AzureSasCredential, TokenCredential
@@ -902,7 +902,13 @@ def download_file(
         :keyword validate_content:
             Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the file.
             Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
-            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+
+            .. note:: When using CRC64 validation (including when "auto" resolves to CRC64):
+
+                - The ``ext-checksums`` extra must be installed.
+                - Automatic decompression is not supported. If ``decompress=True`` is explicitly
+                  set, a :class:`ValueError` will be raised. If ``decompress`` is not specified,
+                  it will be set to ``False`` automatically.
         :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword lease:
             Required if the file has an active lease. Value can be a ShareLeaseClient object
@@ -945,6 +951,13 @@ def download_file(
         access_conditions = get_access_conditions(kwargs.pop("lease", None))
         validate_content = parse_validation_option(kwargs.pop("validate_content", None))
 
+        # Decompression is not supported with CRC64 content validation
+        if is_crc64_validation(validate_content):
+            decompress = kwargs.get("decompress")
+            if decompress is True:
+                raise ValueError("Decompression is not supported when using CRC64 content validation.")
+            kwargs["decompress"] = False
+
         return StorageStreamDownloader(
             client=self._client.file,
             config=self._config,

sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_file_client_async.py

@@ -60,7 +60,7 @@
 from .._shared.request_handlers import add_metadata_headers, get_length
 from .._shared.response_handlers import process_storage_error, return_response_headers
 from .._shared.uploads_async import AsyncIterStreamer, FileChunkUploader, IterStreamer, upload_data_chunks
-from .._shared.validation import CV_TYPE_PARSED, parse_validation_option
+from .._shared.validation import CV_TYPE_PARSED, is_crc64_validation, parse_validation_option
 from ._download_async import StorageStreamDownloader
 from ._lease_async import ShareLeaseClient
 from ._models import FileProperties, Handle, HandlesPaged
@@ -916,7 +916,13 @@ async def download_file(
         :keyword validate_content:
             Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the file.
             Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
-            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+
+            .. note:: When using CRC64 validation (including when "auto" resolves to CRC64):
+
+                - The ``ext-checksums`` extra must be installed.
+                - Automatic decompression is not supported. If ``decompress=True`` is explicitly
+                  set, a :class:`ValueError` will be raised. If ``decompress`` is not specified,
+                  it will be set to ``False`` automatically.
         :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword lease:
             Required if the file has an active lease. Value can be a ShareLeaseClient object
@@ -962,6 +968,13 @@ async def download_file(
         access_conditions = get_access_conditions(kwargs.pop("lease", None))
         validate_content = parse_validation_option(kwargs.pop("validate_content", None))
 
+        # Decompression is not supported with CRC64 content validation
+        if is_crc64_validation(validate_content):
+            decompress = kwargs.get("decompress")
+            if decompress is True:
+                raise ValueError("Decompression is not supported when using CRC64 content validation.")
+            kwargs["decompress"] = False
+
         downloader = StorageStreamDownloader(
             client=self._client.file,
             config=self._config,

sdk/storage/azure-storage-file-share/tests/test_content_validation.py

@@ -275,3 +275,23 @@ def test_download_file_large_chunks(self, **kwargs):
         # Assert
         assert content == data
         assert partial == data[5 * 1024 * 1024 : 30 * 1024 * 1024]
+
+    @FileSharePreparer()
+    @pytest.mark.parametrize("a", ["auto", "crc64"])  # a: validate_content
+    @GenericTestProxyParametrize1()
+    @recorded_by_proxy
+    def test_download_decompress_with_crc64(self, a, **kwargs):
+        storage_account_name = kwargs.pop("storage_account_name")
+
+        self._setup(storage_account_name)
+        file = self.share_client.get_file_client(self._get_file_reference())
+        data = b"abc" * 512
+        file.upload_file(data)
+
+        # decompress=True should raise ValueError
+        with pytest.raises(ValueError, match="Decompression is not supported when using CRC64 content validation."):
+            file.download_file(validate_content=a, decompress=True)
+
+        # decompress=False should work fine
+        downloader = file.download_file(validate_content=a, decompress=False)
+        assert downloader.readall() == data

sdk/storage/azure-storage-file-share/tests/test_content_validation_async.py

@@ -274,3 +274,23 @@ async def test_download_file_large_chunks(self, **kwargs):
         # Assert
         assert content == data
         assert partial == data[5 * 1024 * 1024 : 30 * 1024 * 1024]
+
+    @FileSharePreparer()
+    @pytest.mark.parametrize("a", ["auto", "crc64"])  # a: validate_content
+    @GenericTestProxyParametrize1()
+    @recorded_by_proxy_async
+    async def test_download_decompress_with_crc64(self, a, **kwargs):
+        storage_account_name = kwargs.pop("storage_account_name")
+
+        await self._setup(storage_account_name)
+        file = self.share_client.get_file_client(self._get_file_reference())
+        data = b"abc" * 512
+        await file.upload_file(data)
+
+        # decompress=True should raise ValueError
+        with pytest.raises(ValueError, match="Decompression is not supported when using CRC64 content validation."):
+            await file.download_file(validate_content=a, decompress=True)
+
+        # decompress=False should work fine
+        downloader = await file.download_file(validate_content=a, decompress=False)
+        assert await downloader.readall() == data