Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion virtualizarr/manifests/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ def __init__(
path, offset, length = entry.values()
entry = ChunkEntry.with_validation(path=path, offset=offset, length=length) # type: ignore[attr-defined]

split_key = parse_manifest_index(key, separator)
split_key = () if shape == () else parse_manifest_index(key, separator)
paths[split_key] = entry["path"]
offsets[split_key] = entry["offset"]
lengths[split_key] = entry["length"]
Expand Down
11 changes: 9 additions & 2 deletions virtualizarr/parsers/kerchunk/translator.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,13 @@ def manifestarray_from_kerchunk_refs(
chunk_dict, metadata, zattrs = parse_array_refs(arr_refs)
# we want to remove the _ARRAY_DIMENSIONS from the final variables' .attrs
if chunk_dict:
manifest = manifest_from_kerchunk_chunk_dict(chunk_dict, fs_root=fs_root)
chunk_grid_shape = determine_chunk_grid_shape(
metadata.shape,
metadata.chunks,
)
manifest = manifest_from_kerchunk_chunk_dict(
chunk_dict, fs_root=fs_root, shape=chunk_grid_shape
)
marr = ManifestArray(metadata=metadata, chunkmanifest=manifest)
elif len(metadata.shape) != 0:
# empty variables don't have physical chunks, but zarray shows that the variable
Expand All @@ -202,6 +208,7 @@ def manifestarray_from_kerchunk_refs(
def manifest_from_kerchunk_chunk_dict(
kerchunk_chunk_dict: dict[ChunkKey, str | tuple[str] | tuple[str, int, int]],
fs_root: str | None = None,
shape: tuple[int, ...] | None = None,
) -> ChunkManifest:
"""Create a single ChunkManifest from the mapping of keys to chunk information stored inside kerchunk array refs."""

Expand All @@ -216,7 +223,7 @@ def manifest_from_kerchunk_chunk_dict(
raise TypeError(f"Unexpected type {type(v)} for chunk value: {v}")

chunk_entries[k] = chunkentry_from_kerchunk(v, fs_root=fs_root)
return ChunkManifest(entries=chunk_entries)
return ChunkManifest(entries=chunk_entries, shape=shape)


def chunkentry_from_kerchunk(
Expand Down
2 changes: 1 addition & 1 deletion virtualizarr/tests/test_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def test_kerchunk_roundtrip_in_memory_no_concat(array_v3_metadata):
manifest = ChunkManifest(entries=chunks_dict)
metadata = create_v3_array_metadata(
shape=(2, 4),
chunk_shape=(2, 4),
chunk_shape=(2, 2),
data_type=np.dtype("float32"),
)
marr = ManifestArray(
Expand Down
17 changes: 17 additions & 0 deletions virtualizarr/tests/test_parsers/test_kerchunk.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,23 @@ def test_kerchunk_parquet_sparse_array(tmp_path, local_registry):
assert "1.0" not in manifest


def test_scalar_variable_manifest_shape(refs_file_factory, local_registry):
"""Scalar variables from kerchunk refs should produce a manifest with shape (), not (1,)."""
zarray = '{"chunks":[],"compressor":null,"dtype":"<i4","fill_value":null,"filters":null,"order":"C","shape":[],"zarr_format":2}'
zattrs = '{"_ARRAY_DIMENSIONS":[]}'
refs_file = refs_file_factory(
zarray=zarray,
zattrs=zattrs,
chunks={"a/0": ["/test1.nc", 100, 4]},
)
parser = KerchunkJSONParser()
with open_virtual_dataset(
url=refs_file, registry=local_registry, parser=parser
) as vds:
assert vds["a"].shape == ()
assert vds["a"].data.manifest.shape_chunk_grid == ()


def test_handle_relative_paths(refs_file_factory, local_registry):
# deliberately use relative path here, see https://github.com/zarr-developers/VirtualiZarr/pull/243#issuecomment-2492341326
refs_file = refs_file_factory(chunks={"a/0.0": ["test1.nc", 6144, 48]})
Expand Down
Loading