diff --git a/tests/test_codecs.py b/tests/test_codecs.py index befce8e..e333dd6 100644 --- a/tests/test_codecs.py +++ b/tests/test_codecs.py @@ -5,17 +5,16 @@ import numpy as np import pytest -from zarr import Array, AsyncArray, config +from zarr import Array, AsyncArray, config, create_array +from zarr.api.asynchronous import create_array as create_async_array from zarr.codecs import ( - BytesCodec, - ShardingCodec, TransposeCodec, ) from zarr.core.buffer import default_buffer_prototype +from zarr.core.chunk_key_encodings import ChunkKeyEncodingParams from zarr.storage import StorePath if TYPE_CHECKING: - from zarr.abc.codec import Codec from zarr.abc.store import Store from zarr.core.buffer.core import NDArrayLike from zarr.core.common import MemoryOrder @@ -73,32 +72,17 @@ async def test_order( data = np.arange(0, 256, dtype="uint16").reshape((32, 8), order=input_order) path = "order" spath = StorePath(store, path=path) - codecs_: list[Codec] = ( - [ - ShardingCodec( - chunk_shape=(16, 8), - codecs=[ - TransposeCodec(order=order_from_dim(store_order, data.ndim)), - BytesCodec(), - ], - ) - ] - if with_sharding - else [ - TransposeCodec(order=order_from_dim(store_order, data.ndim)), - BytesCodec(), - ] - ) with config.set({"array.order": runtime_write_order}): - a = await AsyncArray.create( + a = await create_async_array( spath, shape=data.shape, - chunk_shape=(32, 8), + chunks=(32, 8), + shards=(16, 8) if with_sharding else None, dtype=data.dtype, fill_value=0, - chunk_key_encoding=("v2", "."), - codecs=codecs_, + chunk_key_encoding=ChunkKeyEncodingParams(name="v2", separator="."), + filters=[TransposeCodec(order=order_from_dim(store_order, data.ndim))], ) await _AsyncArrayProxy(a)[:, :].set(data) @@ -135,18 +119,15 @@ def test_order_implicit( data = np.arange(0, 256, dtype="uint16").reshape((16, 16), order=input_order) path = "order_implicit" spath = StorePath(store, path) - codecs_: list[Codec] | None = ( - [ShardingCodec(chunk_shape=(8, 8))] if with_sharding else None - ) with config.set({"array.order": runtime_write_order}): - a = Array.create( + a = create_array( spath, shape=data.shape, - chunk_shape=(16, 16), + chunks=(16, 16), + shards=(8, 8) if with_sharding else None, dtype=data.dtype, fill_value=0, - codecs=codecs_, ) a[:, :] = data @@ -167,10 +148,10 @@ def test_order_implicit( def test_write_partial_chunks(store: Store) -> None: data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) spath = StorePath(store) - a = Array.create( + a = create_array( spath, shape=data.shape, - chunk_shape=(20, 20), + chunks=(20, 20), dtype=data.dtype, fill_value=1, ) @@ -182,10 +163,10 @@ async def test_delete_empty_chunks(store: Store) -> None: data = np.ones((16, 16)) path = "delete_empty_chunks" spath = StorePath(store, path) - a = await AsyncArray.create( + a = await create_async_array( spath, shape=data.shape, - chunk_shape=(32, 32), + chunks=(32, 32), dtype=data.dtype, fill_value=1, ) @@ -199,39 +180,38 @@ def test_invalid_metadata(store: Store) -> None: # LD: Disabled for `zarrs`. Including endianness for a single-byte data type is not invalid. # spath2 = StorePath(store, "invalid_endian") # with pytest.raises(TypeError): - # Array.create( + # create_array( # spath2, # shape=(16, 16), - # chunk_shape=(16, 16), + # chunks=(16, 16), # dtype=np.dtype("uint8"), # fill_value=0, - # codecs=[ - # BytesCodec(endian="big"), + # filters=[ # TransposeCodec(order=order_from_dim("F", 2)), # ], + # serializer=BytesCodec(endian="big"), # ) spath3 = StorePath(store, "invalid_order") with pytest.raises(TypeError): - Array.create( + create_array( spath3, shape=(16, 16), - chunk_shape=(16, 16), + chunks=(16, 16), dtype=np.dtype("uint8"), fill_value=0, - codecs=[ - BytesCodec(), + filters=[ TransposeCodec(order="F"), # type: ignore[arg-type] ], ) spath4 = StorePath(store, "invalid_missing_bytes_codec") with pytest.raises(ValueError, match=r".*[Cc]odec.*required"): - Array.create( + create_array( spath4, shape=(16, 16), - chunk_shape=(16, 16), + chunks=(16, 16), dtype=np.dtype("uint8"), fill_value=0, - codecs=[ + filters=[ TransposeCodec(order=order_from_dim("F", 2)), ], ) @@ -239,44 +219,38 @@ def test_invalid_metadata(store: Store) -> None: with pytest.raises( ValueError, match=r".*shard.*chunk_shape.*array.*shape.*need.*same.*dimensions" ): - Array.create( + create_array( spath5, shape=(16, 16), - chunk_shape=(16, 16), + chunks=(16, 16), + shards=(8,), dtype=np.dtype("uint8"), fill_value=0, - codecs=[ - ShardingCodec(chunk_shape=(8,)), - ], ) spath6 = StorePath(store, "invalid_inner_chunk_shape") with pytest.raises( ValueError, match=r".*array.*chunk_shape.*divisible.*shard.*chunk_shape" ): - Array.create( + create_array( spath6, shape=(16, 16), - chunk_shape=(16, 16), + chunks=(16, 16), + shards=(8, 7), dtype=np.dtype("uint8"), fill_value=0, - codecs=[ - ShardingCodec(chunk_shape=(8, 7)), - ], ) # LD: Disabled for `zarrs`. Such checks do not exist. # Also this is not invalid metadata, should be a separate test. # spath7 = StorePath(store, "warning_inefficient_codecs") # with pytest.warns(UserWarning): - # Array.create( + # create_array( # spath7, # shape=(16, 16), - # chunk_shape=(16, 16), + # chunks=(16, 16), + # shards=(8, 8), # dtype=np.dtype("uint8"), # fill_value=0, - # codecs=[ - # ShardingCodec(chunk_shape=(8, 8)), - # GzipCodec(), - # ], + # compressors=[GzipCodec()], # ) @@ -284,12 +258,12 @@ async def test_resize(store: Store) -> None: data = np.zeros((16, 18), dtype="uint16") path = "resize" spath = StorePath(store, path) - a = await AsyncArray.create( + a = await create_async_array( spath, shape=data.shape, - chunk_shape=(10, 10), + chunks=(10, 10), dtype=data.dtype, - chunk_key_encoding=("v2", "."), + chunk_key_encoding=ChunkKeyEncodingParams(name="v2", separator="."), fill_value=1, ) diff --git a/tests/test_endian.py b/tests/test_endian.py index 5353d21..eecbee6 100644 --- a/tests/test_endian.py +++ b/tests/test_endian.py @@ -2,9 +2,10 @@ import numpy as np import pytest -from zarr import AsyncArray from zarr.abc.store import Store +from zarr.api.asynchronous import create_array as create_async_array from zarr.codecs import BytesCodec +from zarr.core.chunk_key_encodings import ChunkKeyEncodingParams from zarr.storage import StorePath from .test_codecs import _AsyncArrayProxy @@ -15,14 +16,14 @@ async def test_endian(store: Store, endian: Literal["big", "little"]) -> None: data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) path = "endian" spath = StorePath(store, path) - a = await AsyncArray.create( + a = await create_async_array( spath, shape=data.shape, - chunk_shape=(16, 16), + chunks=(16, 16), dtype=data.dtype, fill_value=0, - chunk_key_encoding=("v2", "."), - codecs=[BytesCodec(endian=endian)], + chunk_key_encoding=ChunkKeyEncodingParams(name="v2", separator="."), + serializer=BytesCodec(endian=endian), ) await _AsyncArrayProxy(a)[:, :].set(data) @@ -40,14 +41,14 @@ async def test_endian_write( data = np.arange(0, 256, dtype=dtype_input_endian).reshape((16, 16)) path = "endian" spath = StorePath(store, path) - a = await AsyncArray.create( + a = await create_async_array( spath, shape=data.shape, - chunk_shape=(16, 16), + chunks=(16, 16), dtype="uint16", fill_value=0, - chunk_key_encoding=("v2", "."), - codecs=[BytesCodec(endian=dtype_store_endian)], + chunk_key_encoding=ChunkKeyEncodingParams(name="v2", separator="."), + serializer=BytesCodec(endian=dtype_store_endian), ) await _AsyncArrayProxy(a)[:, :].set(data) diff --git a/tests/test_gzip.py b/tests/test_gzip.py index e3a2c26..a0a535f 100644 --- a/tests/test_gzip.py +++ b/tests/test_gzip.py @@ -1,20 +1,20 @@ import numpy as np -from zarr import Array +from zarr import create_array from zarr.abc.store import Store -from zarr.codecs import BytesCodec, GzipCodec +from zarr.codecs import GzipCodec from zarr.storage import StorePath def test_gzip(store: Store) -> None: data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) - a = Array.create( + a = create_array( StorePath(store), shape=data.shape, - chunk_shape=(16, 16), + chunks=(16, 16), dtype=data.dtype, fill_value=0, - codecs=[BytesCodec(), GzipCodec()], + compressors=[GzipCodec()], ) a[:, :] = data diff --git a/tests/test_sharding.py b/tests/test_sharding.py index 599b5e9..2de7e4d 100644 --- a/tests/test_sharding.py +++ b/tests/test_sharding.py @@ -3,15 +3,16 @@ import numpy as np import numpy.typing as npt import pytest -from zarr import Array, AsyncArray +from zarr import create_array from zarr.abc.store import Store +from zarr.api.asynchronous import create_array as create_async_array from zarr.codecs import ( BloscCodec, - BytesCodec, ShardingCodec, ShardingCodecIndexLocation, TransposeCodec, ) +from zarr.core.array import ShardsConfigParam from zarr.core.buffer import default_buffer_prototype from zarr.storage import StorePath @@ -42,23 +43,17 @@ def test_sharding( """ data = array_fixture spath = StorePath(store) - arr = Array.create( + arr = create_array( spath, shape=tuple(s + offset for s in data.shape), - chunk_shape=(64,) * data.ndim, + chunks=(64,) * data.ndim, + shards=ShardsConfigParam( + shape=(32,) * data.ndim, index_location=index_location + ), dtype=data.dtype, fill_value=6, - codecs=[ - ShardingCodec( - chunk_shape=(32,) * data.ndim, - codecs=[ - TransposeCodec(order=order_from_dim("F", data.ndim)), - BytesCodec(), - BloscCodec(cname="lz4"), - ], - index_location=index_location, - ) - ], + filters=[TransposeCodec(order=order_from_dim("F", data.ndim))], + compressors=[BloscCodec(cname="lz4")], ) write_region = tuple(slice(offset, None) for dim in range(data.ndim)) arr[write_region] = data @@ -87,23 +82,15 @@ def test_sharding_partial( ) -> None: data = array_fixture spath = StorePath(store) - a = Array.create( + a = create_array( spath, shape=tuple(a + 10 for a in data.shape), - chunk_shape=(64, 64, 64), + chunks=(64, 64, 64), + shards=ShardsConfigParam(shape=(32, 32, 32), index_location=index_location), dtype=data.dtype, fill_value=0, - codecs=[ - ShardingCodec( - chunk_shape=(32, 32, 32), - codecs=[ - TransposeCodec(order=order_from_dim("F", data.ndim)), - BytesCodec(), - BloscCodec(cname="lz4"), - ], - index_location=index_location, - ) - ], + filters=[TransposeCodec(order=order_from_dim("F", data.ndim))], + compressors=[BloscCodec(cname="lz4")], ) a[10:, 10:, 10:] = data @@ -131,19 +118,15 @@ def test_sharding_partial_readwrite( ) -> None: data = array_fixture spath = StorePath(store) - a = Array.create( + a = create_array( spath, shape=data.shape, - chunk_shape=data.shape, + chunks=data.shape, + shards=ShardsConfigParam( + shape=(1, data.shape[1], data.shape[2]), index_location=index_location + ), dtype=data.dtype, fill_value=0, - codecs=[ - ShardingCodec( - chunk_shape=(1, data.shape[1], data.shape[2]), - codecs=[BytesCodec()], - index_location=index_location, - ) - ], ) a[:] = data @@ -171,23 +154,15 @@ def test_sharding_partial_read( ) -> None: data = array_fixture spath = StorePath(store) - a = Array.create( + a = create_array( spath, shape=tuple(a + 10 for a in data.shape), - chunk_shape=(64, 64, 64), + chunks=(64, 64, 64), + shards=ShardsConfigParam(shape=(32, 32, 32), index_location=index_location), dtype=data.dtype, fill_value=1, - codecs=[ - ShardingCodec( - chunk_shape=(32, 32, 32), - codecs=[ - TransposeCodec(order=order_from_dim("F", data.ndim)), - BytesCodec(), - BloscCodec(cname="lz4"), - ], - index_location=index_location, - ) - ], + filters=[TransposeCodec(order=order_from_dim("F", data.ndim))], + compressors=[BloscCodec(cname="lz4")], ) read_data = a[0:10, 0:10, 0:10] @@ -209,23 +184,15 @@ def test_sharding_partial_overwrite( ) -> None: data = array_fixture[:10, :10, :10] spath = StorePath(store) - a = Array.create( + a = create_array( spath, shape=tuple(a + 10 for a in data.shape), - chunk_shape=(64, 64, 64), + chunks=(64, 64, 64), + shards=ShardsConfigParam(shape=(32, 32, 32), index_location=index_location), dtype=data.dtype, fill_value=1, - codecs=[ - ShardingCodec( - chunk_shape=(32, 32, 32), - codecs=[ - TransposeCodec(order=order_from_dim("F", data.ndim)), - BytesCodec(), - BloscCodec(cname="lz4"), - ], - index_location=index_location, - ) - ], + filters=[TransposeCodec(order=order_from_dim("F", data.ndim))], + compressors=[BloscCodec(cname="lz4")], ) a[:10, :10, :10] = data @@ -262,23 +229,18 @@ def test_nested_sharding( ) -> None: data = array_fixture spath = StorePath(store) - a = Array.create( + a = create_array( spath, shape=data.shape, - chunk_shape=(64, 64, 64), + chunks=(64, 64, 64), + shards=ShardsConfigParam( + shape=(32, 32, 32), index_location=outer_index_location + ), dtype=data.dtype, fill_value=0, - codecs=[ - ShardingCodec( - chunk_shape=(32, 32, 32), - codecs=[ - ShardingCodec( - chunk_shape=(16, 16, 16), index_location=inner_index_location - ) - ], - index_location=outer_index_location, - ) - ], + serializer=ShardingCodec( + chunk_shape=(16, 16, 16), index_location=inner_index_location + ), ) a[:, :, :] = data @@ -291,21 +253,14 @@ def test_nested_sharding( def test_write_partial_sharded_chunks(store: Store) -> None: data = np.arange(0, 16 * 16, dtype="uint16").reshape((16, 16)) spath = StorePath(store) - a = Array.create( + a = create_array( spath, shape=(40, 40), - chunk_shape=(20, 20), + chunks=(20, 20), + shards=(10, 10), dtype=data.dtype, fill_value=1, - codecs=[ - ShardingCodec( - chunk_shape=(10, 10), - codecs=[ - BytesCodec(), - BloscCodec(), - ], - ) - ], + compressors=[BloscCodec()], ) a[0:16, 0:16] = data assert np.array_equal(a[0:16, 0:16], data) @@ -316,13 +271,13 @@ async def test_delete_empty_shards(store: Store) -> None: pytest.skip("store does not support deletes") path = "delete_empty_shards" spath = StorePath(store, path) - a = await AsyncArray.create( + a = create_async_array( spath, shape=(16, 16), - chunk_shape=(8, 16), + chunks=(8, 16), + shards=(8, 8), dtype="uint16", fill_value=1, - codecs=[ShardingCodec(chunk_shape=(8, 8))], ) await _AsyncArrayProxy(a)[:, :].set(np.zeros((16, 16))) await _AsyncArrayProxy(a)[8:, :].set(np.ones((8, 16))) @@ -354,13 +309,13 @@ async def test_sharding_with_empty_inner_chunk( path = f"sharding_with_empty_inner_chunk_{index_location}" spath = StorePath(store, path) - a = await AsyncArray.create( + a = await create_async_array( spath, shape=(16, 16), - chunk_shape=(8, 8), + chunks=(8, 8), + shards=ShardsConfigParam(shape=(4, 4), index_location=index_location), dtype="uint32", fill_value=fill_value, - codecs=[ShardingCodec(chunk_shape=(4, 4), index_location=index_location)], ) data[:4, :4] = fill_value await a.setitem(..., data) diff --git a/tests/test_transpose.py b/tests/test_transpose.py index b9a2859..8a8bcdb 100644 --- a/tests/test_transpose.py +++ b/tests/test_transpose.py @@ -1,18 +1,15 @@ -from typing import TYPE_CHECKING - import numpy as np import pytest -from zarr import Array, AsyncArray, config +from zarr import AsyncArray, config, create_array from zarr.abc.store import Store -from zarr.codecs import BytesCodec, ShardingCodec, TransposeCodec +from zarr.api.asynchronous import create_array as create_async_array +from zarr.codecs import TransposeCodec +from zarr.core.chunk_key_encodings import ChunkKeyEncodingParams from zarr.core.common import MemoryOrder from zarr.storage import StorePath from .test_codecs import _AsyncArrayProxy -if TYPE_CHECKING: - from zarr.abc.codec import Codec - @pytest.mark.parametrize("input_order", ["F", "C"]) @pytest.mark.parametrize("runtime_write_order", ["C"]) @@ -28,25 +25,15 @@ async def test_transpose( ) -> None: data = np.arange(0, 256, dtype="uint16").reshape((1, 32, 8), order=input_order) spath = StorePath(store, path="transpose") - codecs_: list[Codec] = ( - [ - ShardingCodec( - chunk_shape=(1, 16, 8), - codecs=[TransposeCodec(order=(2, 1, 0)), BytesCodec()], - ) - ] - if with_sharding - else [TransposeCodec(order=(2, 1, 0)), BytesCodec()] - ) with config.set({"array.order": runtime_write_order}): - a = await AsyncArray.create( + a = await create_async_array( spath, shape=data.shape, - chunk_shape=(1, 32, 8), + shards=(1, 32, 8) if with_sharding else None, dtype=data.dtype, fill_value=0, - chunk_key_encoding=("v2", "."), - codecs=codecs_, + chunk_key_encoding=ChunkKeyEncodingParams(name="v2", separator="."), + filters=[TransposeCodec(order=(2, 1, 0))], ) await _AsyncArrayProxy(a)[:, :].set(data) @@ -73,13 +60,13 @@ def test_transpose_non_self_inverse(store: Store, order: list[int]) -> None: shape = [i + 3 for i in range(len(order))] data = np.arange(0, np.prod(shape), dtype="uint16").reshape(shape) spath = StorePath(store, "transpose_non_self_inverse") - a = Array.create( + a = create_array( spath, shape=data.shape, - chunk_shape=data.shape, + chunks=data.shape, dtype=data.dtype, fill_value=0, - codecs=[TransposeCodec(order=order), BytesCodec()], + filters=[TransposeCodec(order=order)], ) a[:, :] = data read_data = a[:, :] diff --git a/tests/test_zstd.py b/tests/test_zstd.py index 16d2385..de6a344 100644 --- a/tests/test_zstd.py +++ b/tests/test_zstd.py @@ -1,8 +1,8 @@ import numpy as np import pytest -from zarr import Array +from zarr import create_array from zarr.abc.store import Store -from zarr.codecs import BytesCodec, ZstdCodec +from zarr.codecs import ZstdCodec from zarr.storage import StorePath @@ -10,13 +10,13 @@ def test_zstd(*, store: Store, checksum: bool) -> None: data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) - a = Array.create( + a = create_array( StorePath(store, path="zstd"), shape=data.shape, - chunk_shape=(16, 16), + chunks=(16, 16), dtype=data.dtype, fill_value=0, - codecs=[BytesCodec(), ZstdCodec(level=0, checksum=checksum)], + compressors=[ZstdCodec(level=0, checksum=checksum)], ) a[:, :] = data