Skip to content

chore: upgrade to zarr v3 sharding default#2368

Draft
ilan-gold wants to merge 4 commits intomainfrom
ig/zarr_v3_shard_default
Draft

chore: upgrade to zarr v3 sharding default#2368
ilan-gold wants to merge 4 commits intomainfrom
ig/zarr_v3_shard_default

Conversation

@ilan-gold
Copy link
Contributor

@ilan-gold ilan-gold commented Mar 18, 2026

If this goes in, we will start raising warnings in a "backport" PR for the current default settings

  • Closes #
  • Tests added
  • Release note not necessary because:

@ilan-gold ilan-gold added this to the 0.13.0 milestone Mar 18, 2026
@codecov
Copy link

codecov bot commented Mar 18, 2026

❌ 2 Tests Failed:

Tests completed Failed Passed Skipped
8 2 6 1
View the top 2 failed test(s) by shortest run time
tests.test_dask::test_dask_to_disk_view[zarr2-jax_array]
Stack Traces | 0.001s run time
request = <SubRequest 'diskfmt' for <Function test_dask_to_disk_view[zarr2-jax_array]>>

    #x1B[0m#x1B[37m@pytest#x1B[39;49;00m.fixture(#x1B[90m#x1B[39;49;00m
        params=[(#x1B[33m"#x1B[39;49;00m#x1B[33mh5ad#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m, #x1B[94mNone#x1B[39;49;00m), (#x1B[33m"#x1B[39;49;00m#x1B[33mzarr#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m, #x1B[94m2#x1B[39;49;00m), (#x1B[33m"#x1B[39;49;00m#x1B[33mzarr#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m, #x1B[94m3#x1B[39;49;00m)],#x1B[90m#x1B[39;49;00m
        ids=[#x1B[33m"#x1B[39;49;00m#x1B[33mh5ad#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m, #x1B[33m"#x1B[39;49;00m#x1B[33mzarr2#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m, #x1B[33m"#x1B[39;49;00m#x1B[33mzarr3#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m],#x1B[90m#x1B[39;49;00m
    )#x1B[90m#x1B[39;49;00m
    #x1B[94mdef#x1B[39;49;00m#x1B[90m #x1B[39;49;00m#x1B[92mdiskfmt#x1B[39;49;00m(#x1B[90m#x1B[39;49;00m
        request: pytest.FixtureRequest,#x1B[90m#x1B[39;49;00m
    ) -> Generator[Literal[#x1B[33m"#x1B[39;49;00m#x1B[33mh5ad#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m, #x1B[33m"#x1B[39;49;00m#x1B[33mzarr#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m], #x1B[94mNone#x1B[39;49;00m, #x1B[94mNone#x1B[39;49;00m]:#x1B[90m#x1B[39;49;00m
        #x1B[94mif#x1B[39;49;00m (fmt := request.param[#x1B[94m0#x1B[39;49;00m]) == #x1B[33m"#x1B[39;49;00m#x1B[33mh5ad#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m:#x1B[90m#x1B[39;49;00m
            #x1B[94myield#x1B[39;49;00m fmt#x1B[90m#x1B[39;49;00m
        #x1B[94melse#x1B[39;49;00m:#x1B[90m#x1B[39;49;00m
>           #x1B[94mwith#x1B[39;49;00m ad.settings.override(#x1B[90m#x1B[39;49;00m
                auto_shard_zarr_v3=request.param[#x1B[94m1#x1B[39;49;00m] == #x1B[94m3#x1B[39;49;00m, zarr_write_format=request.param[#x1B[94m1#x1B[39;49;00m]#x1B[90m#x1B[39;49;00m
            ):#x1B[90m#x1B[39;49;00m

#x1B[1m#x1B[31mtests/conftest.py#x1B[0m:46: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
#x1B[1m#x1B[31m../...../_temp/uv-python-dir/cpython-3.14.3-linux-x86_64-gnu/lib/python3.14/contextlib.py#x1B[0m:141: in __enter__
    #x1B[0m#x1B[94mreturn#x1B[39;49;00m #x1B[96mnext#x1B[39;49;00m(#x1B[96mself#x1B[39;49;00m.gen)#x1B[90m#x1B[39;49;00m
           ^^^^^^^^^^^^^^#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[31msrc/anndata/_settings.py#x1B[0m:376: in override
    #x1B[0m#x1B[96msetattr#x1B[39;49;00m(#x1B[96mself#x1B[39;49;00m, k, overrides.get(k))#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[31msrc/anndata/_settings.py#x1B[0m:315: in __setattr__
    #x1B[0mregistered_option.validate(val, #x1B[96mself#x1B[39;49;00m)#x1B[90m#x1B[39;49;00m
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

format = 2
settings = SettingsManager(
	remove_unused_categories=True,
	check_uniqueness=True,
	allow_write_nullable_strings=None,
	zarr_wri..._h5ad=False,
	write_csr_csc_indices_with_min_possible_dtype=False,
	auto_shard_zarr_v3=True,
	copy_on_write_X=False,

)

    #x1B[0m#x1B[94mdef#x1B[39;49;00m#x1B[90m #x1B[39;49;00m#x1B[92mvalidate_zarr_write_format#x1B[39;49;00m(#x1B[96mformat#x1B[39;49;00m: #x1B[96mint#x1B[39;49;00m, settings: SettingsManager):#x1B[90m#x1B[39;49;00m
        validate_int(#x1B[96mformat#x1B[39;49;00m, settings)#x1B[90m#x1B[39;49;00m
        #x1B[94mif#x1B[39;49;00m #x1B[96mformat#x1B[39;49;00m #x1B[95mnot#x1B[39;49;00m #x1B[95min#x1B[39;49;00m {#x1B[94m2#x1B[39;49;00m, #x1B[94m3#x1B[39;49;00m}:#x1B[90m#x1B[39;49;00m
            msg = #x1B[33m"#x1B[39;49;00m#x1B[33mnon-v2 zarr on-disk format not supported#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
            #x1B[94mraise#x1B[39;49;00m #x1B[96mValueError#x1B[39;49;00m(msg)#x1B[90m#x1B[39;49;00m
        #x1B[94mif#x1B[39;49;00m #x1B[96mformat#x1B[39;49;00m == #x1B[94m2#x1B[39;49;00m #x1B[95mand#x1B[39;49;00m #x1B[96mgetattr#x1B[39;49;00m(settings, #x1B[33m"#x1B[39;49;00m#x1B[33mauto_shard_zarr_v3#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m, #x1B[94mFalse#x1B[39;49;00m):#x1B[90m#x1B[39;49;00m
            msg = #x1B[33m"#x1B[39;49;00m#x1B[33mCannot set `zarr_write_format` to 2 with autosharding on.  Please set to `False` `anndata.settings.auto_shard_zarr_v3`#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
>           #x1B[94mraise#x1B[39;49;00m #x1B[96mValueError#x1B[39;49;00m(msg)#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[31mE           ValueError: Cannot set `zarr_write_format` to 2 with autosharding on.  Please set to `False` `anndata.settings.auto_shard_zarr_v3`#x1B[0m

#x1B[1m#x1B[31msrc/anndata/_settings.py#x1B[0m:461: ValueError
tests.test_readwrite::test_readwrite_roundtrip[zarr2-jax.array]
Stack Traces | 0.001s run time
request = <SubRequest 'diskfmt' for <Function test_readwrite_roundtrip[zarr2-jax.array]>>

    #x1B[0m#x1B[37m@pytest#x1B[39;49;00m.fixture(#x1B[90m#x1B[39;49;00m
        params=[(#x1B[33m"#x1B[39;49;00m#x1B[33mh5ad#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m, #x1B[94mNone#x1B[39;49;00m), (#x1B[33m"#x1B[39;49;00m#x1B[33mzarr#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m, #x1B[94m2#x1B[39;49;00m), (#x1B[33m"#x1B[39;49;00m#x1B[33mzarr#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m, #x1B[94m3#x1B[39;49;00m)],#x1B[90m#x1B[39;49;00m
        ids=[#x1B[33m"#x1B[39;49;00m#x1B[33mh5ad#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m, #x1B[33m"#x1B[39;49;00m#x1B[33mzarr2#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m, #x1B[33m"#x1B[39;49;00m#x1B[33mzarr3#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m],#x1B[90m#x1B[39;49;00m
    )#x1B[90m#x1B[39;49;00m
    #x1B[94mdef#x1B[39;49;00m#x1B[90m #x1B[39;49;00m#x1B[92mdiskfmt#x1B[39;49;00m(#x1B[90m#x1B[39;49;00m
        request: pytest.FixtureRequest,#x1B[90m#x1B[39;49;00m
    ) -> Generator[Literal[#x1B[33m"#x1B[39;49;00m#x1B[33mh5ad#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m, #x1B[33m"#x1B[39;49;00m#x1B[33mzarr#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m], #x1B[94mNone#x1B[39;49;00m, #x1B[94mNone#x1B[39;49;00m]:#x1B[90m#x1B[39;49;00m
        #x1B[94mif#x1B[39;49;00m (fmt := request.param[#x1B[94m0#x1B[39;49;00m]) == #x1B[33m"#x1B[39;49;00m#x1B[33mh5ad#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m:#x1B[90m#x1B[39;49;00m
            #x1B[94myield#x1B[39;49;00m fmt#x1B[90m#x1B[39;49;00m
        #x1B[94melse#x1B[39;49;00m:#x1B[90m#x1B[39;49;00m
>           #x1B[94mwith#x1B[39;49;00m ad.settings.override(#x1B[90m#x1B[39;49;00m
                auto_shard_zarr_v3=request.param[#x1B[94m1#x1B[39;49;00m] == #x1B[94m3#x1B[39;49;00m, zarr_write_format=request.param[#x1B[94m1#x1B[39;49;00m]#x1B[90m#x1B[39;49;00m
            ):#x1B[90m#x1B[39;49;00m

#x1B[1m#x1B[31mtests/conftest.py#x1B[0m:46: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
#x1B[1m#x1B[31m../...../_temp/uv-python-dir/cpython-3.14.3-linux-x86_64-gnu/lib/python3.14/contextlib.py#x1B[0m:141: in __enter__
    #x1B[0m#x1B[94mreturn#x1B[39;49;00m #x1B[96mnext#x1B[39;49;00m(#x1B[96mself#x1B[39;49;00m.gen)#x1B[90m#x1B[39;49;00m
           ^^^^^^^^^^^^^^#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[31msrc/anndata/_settings.py#x1B[0m:376: in override
    #x1B[0m#x1B[96msetattr#x1B[39;49;00m(#x1B[96mself#x1B[39;49;00m, k, overrides.get(k))#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[31msrc/anndata/_settings.py#x1B[0m:315: in __setattr__
    #x1B[0mregistered_option.validate(val, #x1B[96mself#x1B[39;49;00m)#x1B[90m#x1B[39;49;00m
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

format = 2
settings = SettingsManager(
	remove_unused_categories=True,
	check_uniqueness=True,
	allow_write_nullable_strings=None,
	zarr_wri..._h5ad=False,
	write_csr_csc_indices_with_min_possible_dtype=False,
	auto_shard_zarr_v3=True,
	copy_on_write_X=False,

)

    #x1B[0m#x1B[94mdef#x1B[39;49;00m#x1B[90m #x1B[39;49;00m#x1B[92mvalidate_zarr_write_format#x1B[39;49;00m(#x1B[96mformat#x1B[39;49;00m: #x1B[96mint#x1B[39;49;00m, settings: SettingsManager):#x1B[90m#x1B[39;49;00m
        validate_int(#x1B[96mformat#x1B[39;49;00m, settings)#x1B[90m#x1B[39;49;00m
        #x1B[94mif#x1B[39;49;00m #x1B[96mformat#x1B[39;49;00m #x1B[95mnot#x1B[39;49;00m #x1B[95min#x1B[39;49;00m {#x1B[94m2#x1B[39;49;00m, #x1B[94m3#x1B[39;49;00m}:#x1B[90m#x1B[39;49;00m
            msg = #x1B[33m"#x1B[39;49;00m#x1B[33mnon-v2 zarr on-disk format not supported#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
            #x1B[94mraise#x1B[39;49;00m #x1B[96mValueError#x1B[39;49;00m(msg)#x1B[90m#x1B[39;49;00m
        #x1B[94mif#x1B[39;49;00m #x1B[96mformat#x1B[39;49;00m == #x1B[94m2#x1B[39;49;00m #x1B[95mand#x1B[39;49;00m #x1B[96mgetattr#x1B[39;49;00m(settings, #x1B[33m"#x1B[39;49;00m#x1B[33mauto_shard_zarr_v3#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m, #x1B[94mFalse#x1B[39;49;00m):#x1B[90m#x1B[39;49;00m
            msg = #x1B[33m"#x1B[39;49;00m#x1B[33mCannot set `zarr_write_format` to 2 with autosharding on.  Please set to `False` `anndata.settings.auto_shard_zarr_v3`#x1B[39;49;00m#x1B[33m"#x1B[39;49;00m#x1B[90m#x1B[39;49;00m
>           #x1B[94mraise#x1B[39;49;00m #x1B[96mValueError#x1B[39;49;00m(msg)#x1B[90m#x1B[39;49;00m
#x1B[1m#x1B[31mE           ValueError: Cannot set `zarr_write_format` to 2 with autosharding on.  Please set to `False` `anndata.settings.auto_shard_zarr_v3`#x1B[0m

#x1B[1m#x1B[31msrc/anndata/_settings.py#x1B[0m:461: ValueError

To view more test analytics, go to the Test Analytics Dashboard
📋 Got 3 mins? Take this short survey to help us improve Test Analytics.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Projects

None yet

Development

Successfully merging this pull request may close these issues.

1 participant