Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 38 additions & 64 deletions tests/test_codecs.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,16 @@

import numpy as np
import pytest
from zarr import Array, AsyncArray, config
from zarr import Array, AsyncArray, config, create_array
from zarr.api.asynchronous import create_array as create_async_array
from zarr.codecs import (
BytesCodec,
ShardingCodec,
TransposeCodec,
)
from zarr.core.buffer import default_buffer_prototype
from zarr.core.chunk_key_encodings import ChunkKeyEncodingParams
from zarr.storage import StorePath

if TYPE_CHECKING:
from zarr.abc.codec import Codec
from zarr.abc.store import Store
from zarr.core.buffer.core import NDArrayLike
from zarr.core.common import MemoryOrder
Expand Down Expand Up @@ -73,32 +72,17 @@ async def test_order(
data = np.arange(0, 256, dtype="uint16").reshape((32, 8), order=input_order)
path = "order"
spath = StorePath(store, path=path)
codecs_: list[Codec] = (
[
ShardingCodec(
chunk_shape=(16, 8),
codecs=[
TransposeCodec(order=order_from_dim(store_order, data.ndim)),
BytesCodec(),
],
)
]
if with_sharding
else [
TransposeCodec(order=order_from_dim(store_order, data.ndim)),
BytesCodec(),
]
)

with config.set({"array.order": runtime_write_order}):
a = await AsyncArray.create(
a = await create_async_array(
spath,
shape=data.shape,
chunk_shape=(32, 8),
chunks=(32, 8),
shards=(16, 8) if with_sharding else None,
dtype=data.dtype,
fill_value=0,
chunk_key_encoding=("v2", "."),
codecs=codecs_,
chunk_key_encoding=ChunkKeyEncodingParams(name="v2", separator="."),
filters=[TransposeCodec(order=order_from_dim(store_order, data.ndim))],
)

await _AsyncArrayProxy(a)[:, :].set(data)
Expand Down Expand Up @@ -135,18 +119,15 @@ def test_order_implicit(
data = np.arange(0, 256, dtype="uint16").reshape((16, 16), order=input_order)
path = "order_implicit"
spath = StorePath(store, path)
codecs_: list[Codec] | None = (
[ShardingCodec(chunk_shape=(8, 8))] if with_sharding else None
)

with config.set({"array.order": runtime_write_order}):
a = Array.create(
a = create_array(
spath,
shape=data.shape,
chunk_shape=(16, 16),
chunks=(16, 16),
shards=(8, 8) if with_sharding else None,
dtype=data.dtype,
fill_value=0,
codecs=codecs_,
)

a[:, :] = data
Expand All @@ -167,10 +148,10 @@ def test_order_implicit(
def test_write_partial_chunks(store: Store) -> None:
data = np.arange(0, 256, dtype="uint16").reshape((16, 16))
spath = StorePath(store)
a = Array.create(
a = create_array(
spath,
shape=data.shape,
chunk_shape=(20, 20),
chunks=(20, 20),
dtype=data.dtype,
fill_value=1,
)
Expand All @@ -182,10 +163,10 @@ async def test_delete_empty_chunks(store: Store) -> None:
data = np.ones((16, 16))
path = "delete_empty_chunks"
spath = StorePath(store, path)
a = await AsyncArray.create(
a = await create_async_array(
spath,
shape=data.shape,
chunk_shape=(32, 32),
chunks=(32, 32),
dtype=data.dtype,
fill_value=1,
)
Expand All @@ -199,97 +180,90 @@ def test_invalid_metadata(store: Store) -> None:
# LD: Disabled for `zarrs`. Including endianness for a single-byte data type is not invalid.
# spath2 = StorePath(store, "invalid_endian")
# with pytest.raises(TypeError):
# Array.create(
# create_array(
# spath2,
# shape=(16, 16),
# chunk_shape=(16, 16),
# chunks=(16, 16),
# dtype=np.dtype("uint8"),
# fill_value=0,
# codecs=[
# BytesCodec(endian="big"),
# filters=[
# TransposeCodec(order=order_from_dim("F", 2)),
# ],
# serializer=BytesCodec(endian="big"),
# )
spath3 = StorePath(store, "invalid_order")
with pytest.raises(TypeError):
Array.create(
create_array(
spath3,
shape=(16, 16),
chunk_shape=(16, 16),
chunks=(16, 16),
dtype=np.dtype("uint8"),
fill_value=0,
codecs=[
BytesCodec(),
filters=[
TransposeCodec(order="F"), # type: ignore[arg-type]
],
)
spath4 = StorePath(store, "invalid_missing_bytes_codec")
with pytest.raises(ValueError, match=r".*[Cc]odec.*required"):
Array.create(
create_array(
spath4,
shape=(16, 16),
chunk_shape=(16, 16),
chunks=(16, 16),
dtype=np.dtype("uint8"),
fill_value=0,
codecs=[
filters=[
TransposeCodec(order=order_from_dim("F", 2)),
],
)
spath5 = StorePath(store, "invalid_inner_chunk_shape")
with pytest.raises(
ValueError, match=r".*shard.*chunk_shape.*array.*shape.*need.*same.*dimensions"
):
Array.create(
create_array(
spath5,
shape=(16, 16),
chunk_shape=(16, 16),
chunks=(16, 16),
shards=(8,),
dtype=np.dtype("uint8"),
fill_value=0,
codecs=[
ShardingCodec(chunk_shape=(8,)),
],
)
spath6 = StorePath(store, "invalid_inner_chunk_shape")
with pytest.raises(
ValueError, match=r".*array.*chunk_shape.*divisible.*shard.*chunk_shape"
):
Array.create(
create_array(
spath6,
shape=(16, 16),
chunk_shape=(16, 16),
chunks=(16, 16),
shards=(8, 7),
dtype=np.dtype("uint8"),
fill_value=0,
codecs=[
ShardingCodec(chunk_shape=(8, 7)),
],
)
# LD: Disabled for `zarrs`. Such checks do not exist.
# Also this is not invalid metadata, should be a separate test.
# spath7 = StorePath(store, "warning_inefficient_codecs")
# with pytest.warns(UserWarning):
# Array.create(
# create_array(
# spath7,
# shape=(16, 16),
# chunk_shape=(16, 16),
# chunks=(16, 16),
# shards=(8, 8),
# dtype=np.dtype("uint8"),
# fill_value=0,
# codecs=[
# ShardingCodec(chunk_shape=(8, 8)),
# GzipCodec(),
# ],
# compressors=[GzipCodec()],
# )


async def test_resize(store: Store) -> None:
data = np.zeros((16, 18), dtype="uint16")
path = "resize"
spath = StorePath(store, path)
a = await AsyncArray.create(
a = await create_async_array(
spath,
shape=data.shape,
chunk_shape=(10, 10),
chunks=(10, 10),
dtype=data.dtype,
chunk_key_encoding=("v2", "."),
chunk_key_encoding=ChunkKeyEncodingParams(name="v2", separator="."),
fill_value=1,
)

Expand Down
19 changes: 10 additions & 9 deletions tests/test_endian.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@

import numpy as np
import pytest
from zarr import AsyncArray
from zarr.abc.store import Store
from zarr.api.asynchronous import create_array as create_async_array
from zarr.codecs import BytesCodec
from zarr.core.chunk_key_encodings import ChunkKeyEncodingParams
from zarr.storage import StorePath

from .test_codecs import _AsyncArrayProxy
Expand All @@ -15,14 +16,14 @@ async def test_endian(store: Store, endian: Literal["big", "little"]) -> None:
data = np.arange(0, 256, dtype="uint16").reshape((16, 16))
path = "endian"
spath = StorePath(store, path)
a = await AsyncArray.create(
a = await create_async_array(
spath,
shape=data.shape,
chunk_shape=(16, 16),
chunks=(16, 16),
dtype=data.dtype,
fill_value=0,
chunk_key_encoding=("v2", "."),
codecs=[BytesCodec(endian=endian)],
chunk_key_encoding=ChunkKeyEncodingParams(name="v2", separator="."),
serializer=BytesCodec(endian=endian),
)

await _AsyncArrayProxy(a)[:, :].set(data)
Expand All @@ -40,14 +41,14 @@ async def test_endian_write(
data = np.arange(0, 256, dtype=dtype_input_endian).reshape((16, 16))
path = "endian"
spath = StorePath(store, path)
a = await AsyncArray.create(
a = await create_async_array(
spath,
shape=data.shape,
chunk_shape=(16, 16),
chunks=(16, 16),
dtype="uint16",
fill_value=0,
chunk_key_encoding=("v2", "."),
codecs=[BytesCodec(endian=dtype_store_endian)],
chunk_key_encoding=ChunkKeyEncodingParams(name="v2", separator="."),
serializer=BytesCodec(endian=dtype_store_endian),
)

await _AsyncArrayProxy(a)[:, :].set(data)
Expand Down
10 changes: 5 additions & 5 deletions tests/test_gzip.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
import numpy as np
from zarr import Array
from zarr import create_array
from zarr.abc.store import Store
from zarr.codecs import BytesCodec, GzipCodec
from zarr.codecs import GzipCodec
from zarr.storage import StorePath


def test_gzip(store: Store) -> None:
data = np.arange(0, 256, dtype="uint16").reshape((16, 16))

a = Array.create(
a = create_array(
StorePath(store),
shape=data.shape,
chunk_shape=(16, 16),
chunks=(16, 16),
dtype=data.dtype,
fill_value=0,
codecs=[BytesCodec(), GzipCodec()],
compressors=[GzipCodec()],
)

a[:, :] = data
Expand Down
Loading
Loading