diff --git a/sdk/storage/azure-storage-blob/assets.json b/sdk/storage/azure-storage-blob/assets.json index 93dd0f1e298c..30801f8d6ef6 100644 --- a/sdk/storage/azure-storage-blob/assets.json +++ b/sdk/storage/azure-storage-blob/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "python", "TagPrefix": "python/storage/azure-storage-blob", - "Tag": "python/storage/azure-storage-blob_b09e37b521" + "Tag": "python/storage/azure-storage-blob_2d1cd89589" } diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 4eeabfac83b0..19eaec94e03d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -678,7 +678,13 @@ def download_blob( :keyword validate_content: Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob. Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated. - NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed. + + .. note:: When using CRC64 validation (including when "auto" resolves to CRC64): + + - The ``ext-checksums`` extra must be installed. + - Automatic decompression is not supported. If ``decompress=True`` is explicitly + set, a :class:`ValueError` will be raised. If ``decompress`` is not specified, + it will be set to ``False`` automatically. :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']] :keyword lease: Required if the blob has an active lease. If specified, download_blob only diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 1dcdca302943..e2cc5165716c 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -63,7 +63,7 @@ from ._shared.response_handlers import return_headers_and_deserialized, return_response_headers from ._shared.uploads import IterStreamer from ._shared.uploads_async import AsyncIterStreamer -from ._shared.validation import CV_TYPE_PARSED, parse_validation_option +from ._shared.validation import CV_TYPE_PARSED, is_crc64_validation, parse_validation_option from ._upload_helpers import _any_conditions if TYPE_CHECKING: @@ -294,6 +294,13 @@ def _download_blob_options( config.user_agent_policy.user_agent, sdk_moniker, encryption_options["version"], kwargs ) + # Decompression is not supported with CRC64 content validation + if is_crc64_validation(validate_content): + decompress = kwargs.get("decompress") + if decompress is True: + raise ValueError("Decompression is not supported when using CRC64 content validation.") + kwargs["decompress"] = False + options = { "clients": client, "config": config, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 44c63abee2e0..8bf1c87ae35a 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -707,7 +707,13 @@ async def download_blob( :keyword validate_content: Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob. Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated. - NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed. + + .. note:: When using CRC64 validation (including when "auto" resolves to CRC64): + + - The ``ext-checksums`` extra must be installed. + - Automatic decompression is not supported. If ``decompress=True`` is explicitly + set, a :class:`ValueError` will be raised. If ``decompress`` is not specified, + it will be set to ``False`` automatically. :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']] :keyword lease: Required if the blob has an active lease. If specified, download_blob only diff --git a/sdk/storage/azure-storage-blob/tests/test_content_validation.py b/sdk/storage/azure-storage-blob/tests/test_content_validation.py index c4327291e3e0..bc7633b087d3 100644 --- a/sdk/storage/azure-storage-blob/tests/test_content_validation.py +++ b/sdk/storage/azure-storage-blob/tests/test_content_validation.py @@ -645,3 +645,23 @@ def hook_fail_once(response): blob.commit_block_list([BlobBlock("1")]) result = blob.download_blob() assert result.read() == content + + @BlobPreparer() + @pytest.mark.parametrize("a", ["auto", "crc64"]) # a: validate_content + @GenericTestProxyParametrize1() + @recorded_by_proxy + def test_download_decompress_with_crc64(self, a, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + + self._setup(storage_account_name) + blob = self.container.get_blob_client(self._get_blob_reference()) + data = b"abc" * 512 + blob.upload_blob(data, overwrite=True) + + # decompress=True should raise ValueError + with pytest.raises(ValueError, match="Decompression is not supported when using CRC64 content validation."): + blob.download_blob(validate_content=a, decompress=True) + + # decompress=False should work fine + downloader = blob.download_blob(validate_content=a, decompress=False) + assert downloader.read() == data diff --git a/sdk/storage/azure-storage-blob/tests/test_content_validation_async.py b/sdk/storage/azure-storage-blob/tests/test_content_validation_async.py index e22a03b1fe75..00443d20cb03 100644 --- a/sdk/storage/azure-storage-blob/tests/test_content_validation_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_content_validation_async.py @@ -628,3 +628,23 @@ def hook_fail_once(response): await blob.commit_block_list([BlobBlock("1")]) result = await blob.download_blob() assert await result.read() == content + + @BlobPreparer() + @pytest.mark.parametrize("a", ["auto", "crc64"]) # a: validate_content + @GenericTestProxyParametrize1() + @recorded_by_proxy_async + async def test_download_decompress_with_crc64(self, a, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + + await self._setup(storage_account_name) + blob = self.container.get_blob_client(self._get_blob_reference()) + data = b"abc" * 512 + await blob.upload_blob(data, overwrite=True) + + # decompress=True should raise ValueError + with pytest.raises(ValueError, match="Decompression is not supported when using CRC64 content validation."): + await blob.download_blob(validate_content=a, decompress=True) + + # decompress=False should work fine + downloader = await blob.download_blob(validate_content=a, decompress=False) + assert await downloader.read() == data diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py index 6a35eeec7e5b..bef7be8af4eb 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py @@ -688,7 +688,13 @@ def download_file( :keyword validate_content: Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob. Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated. - NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed. + + .. note:: When using CRC64 validation (including when "auto" resolves to CRC64): + + - The ``ext-checksums`` extra must be installed. + - Automatic decompression is not supported. If ``decompress=True`` is explicitly + set, a :class:`ValueError` will be raised. If ``decompress`` is not specified, + it will be set to ``False`` automatically. :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']] :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.py index 835c9dbd53aa..ba0b2b4018c4 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.py @@ -710,7 +710,13 @@ async def download_file( :keyword validate_content: Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob. Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated. - NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed. + + .. note:: When using CRC64 validation (including when "auto" resolves to CRC64): + + - The ``ext-checksums`` extra must be installed. + - Automatic decompression is not supported. If ``decompress=True`` is explicitly + set, a :class:`ValueError` will be raised. If ``decompress`` is not specified, + it will be set to ``False`` automatically. :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']] :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see diff --git a/sdk/storage/azure-storage-file-share/assets.json b/sdk/storage/azure-storage-file-share/assets.json index 79ed1f733678..7dba5a9f6b72 100644 --- a/sdk/storage/azure-storage-file-share/assets.json +++ b/sdk/storage/azure-storage-file-share/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "python", "TagPrefix": "python/storage/azure-storage-file-share", - "Tag": "python/storage/azure-storage-file-share_d5f376c42f" + "Tag": "python/storage/azure-storage-file-share_bcf00830c4" } diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_file_client.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_file_client.py index b730483fdbb1..ba589219fe33 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_file_client.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_file_client.py @@ -42,7 +42,7 @@ from ._shared.request_handlers import add_metadata_headers, get_length from ._shared.response_handlers import return_response_headers, process_storage_error from ._shared.uploads import IterStreamer, FileChunkUploader, upload_data_chunks -from ._shared.validation import CV_TYPE_PARSED, parse_validation_option +from ._shared.validation import CV_TYPE_PARSED, is_crc64_validation, parse_validation_option if TYPE_CHECKING: from azure.core.credentials import AzureNamedKeyCredential, AzureSasCredential, TokenCredential @@ -902,7 +902,13 @@ def download_file( :keyword validate_content: Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the file. Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated. - NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed. + + .. note:: When using CRC64 validation (including when "auto" resolves to CRC64): + + - The ``ext-checksums`` extra must be installed. + - Automatic decompression is not supported. If ``decompress=True`` is explicitly + set, a :class:`ValueError` will be raised. If ``decompress`` is not specified, + it will be set to ``False`` automatically. :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']] :keyword lease: Required if the file has an active lease. Value can be a ShareLeaseClient object @@ -945,6 +951,13 @@ def download_file( access_conditions = get_access_conditions(kwargs.pop("lease", None)) validate_content = parse_validation_option(kwargs.pop("validate_content", None)) + # Decompression is not supported with CRC64 content validation + if is_crc64_validation(validate_content): + decompress = kwargs.get("decompress") + if decompress is True: + raise ValueError("Decompression is not supported when using CRC64 content validation.") + kwargs["decompress"] = False + return StorageStreamDownloader( client=self._client.file, config=self._config, diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_file_client_async.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_file_client_async.py index b6abf4b55718..6b01b33889a0 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_file_client_async.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_file_client_async.py @@ -60,7 +60,7 @@ from .._shared.request_handlers import add_metadata_headers, get_length from .._shared.response_handlers import process_storage_error, return_response_headers from .._shared.uploads_async import AsyncIterStreamer, FileChunkUploader, IterStreamer, upload_data_chunks -from .._shared.validation import CV_TYPE_PARSED, parse_validation_option +from .._shared.validation import CV_TYPE_PARSED, is_crc64_validation, parse_validation_option from ._download_async import StorageStreamDownloader from ._lease_async import ShareLeaseClient from ._models import FileProperties, Handle, HandlesPaged @@ -916,7 +916,13 @@ async def download_file( :keyword validate_content: Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the file. Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated. - NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed. + + .. note:: When using CRC64 validation (including when "auto" resolves to CRC64): + + - The ``ext-checksums`` extra must be installed. + - Automatic decompression is not supported. If ``decompress=True`` is explicitly + set, a :class:`ValueError` will be raised. If ``decompress`` is not specified, + it will be set to ``False`` automatically. :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']] :keyword lease: Required if the file has an active lease. Value can be a ShareLeaseClient object @@ -962,6 +968,13 @@ async def download_file( access_conditions = get_access_conditions(kwargs.pop("lease", None)) validate_content = parse_validation_option(kwargs.pop("validate_content", None)) + # Decompression is not supported with CRC64 content validation + if is_crc64_validation(validate_content): + decompress = kwargs.get("decompress") + if decompress is True: + raise ValueError("Decompression is not supported when using CRC64 content validation.") + kwargs["decompress"] = False + downloader = StorageStreamDownloader( client=self._client.file, config=self._config, diff --git a/sdk/storage/azure-storage-file-share/tests/test_content_validation.py b/sdk/storage/azure-storage-file-share/tests/test_content_validation.py index 6afdfa1194ac..8e2c06543bed 100644 --- a/sdk/storage/azure-storage-file-share/tests/test_content_validation.py +++ b/sdk/storage/azure-storage-file-share/tests/test_content_validation.py @@ -275,3 +275,23 @@ def test_download_file_large_chunks(self, **kwargs): # Assert assert content == data assert partial == data[5 * 1024 * 1024 : 30 * 1024 * 1024] + + @FileSharePreparer() + @pytest.mark.parametrize("a", ["auto", "crc64"]) # a: validate_content + @GenericTestProxyParametrize1() + @recorded_by_proxy + def test_download_decompress_with_crc64(self, a, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + + self._setup(storage_account_name) + file = self.share_client.get_file_client(self._get_file_reference()) + data = b"abc" * 512 + file.upload_file(data) + + # decompress=True should raise ValueError + with pytest.raises(ValueError, match="Decompression is not supported when using CRC64 content validation."): + file.download_file(validate_content=a, decompress=True) + + # decompress=False should work fine + downloader = file.download_file(validate_content=a, decompress=False) + assert downloader.readall() == data diff --git a/sdk/storage/azure-storage-file-share/tests/test_content_validation_async.py b/sdk/storage/azure-storage-file-share/tests/test_content_validation_async.py index 628356cbca07..c9830395d1c6 100644 --- a/sdk/storage/azure-storage-file-share/tests/test_content_validation_async.py +++ b/sdk/storage/azure-storage-file-share/tests/test_content_validation_async.py @@ -274,3 +274,23 @@ async def test_download_file_large_chunks(self, **kwargs): # Assert assert content == data assert partial == data[5 * 1024 * 1024 : 30 * 1024 * 1024] + + @FileSharePreparer() + @pytest.mark.parametrize("a", ["auto", "crc64"]) # a: validate_content + @GenericTestProxyParametrize1() + @recorded_by_proxy_async + async def test_download_decompress_with_crc64(self, a, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + + await self._setup(storage_account_name) + file = self.share_client.get_file_client(self._get_file_reference()) + data = b"abc" * 512 + await file.upload_file(data) + + # decompress=True should raise ValueError + with pytest.raises(ValueError, match="Decompression is not supported when using CRC64 content validation."): + await file.download_file(validate_content=a, decompress=True) + + # decompress=False should work fine + downloader = await file.download_file(validate_content=a, decompress=False) + assert await downloader.readall() == data