From bd4f102fb3726d786d9db4070a20579a861d8f0c Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Thu, 24 Apr 2025 18:24:12 +0000 Subject: [PATCH 01/55] Start work on a libpressio codec wrapper --- Cargo.toml | 2 + codecs/pressio/Cargo.toml | 27 +++++++++++++ codecs/pressio/LICENSE | 1 + codecs/pressio/README.md | 38 ++++++++++++++++++ codecs/pressio/src/lib.rs | 82 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 150 insertions(+) create mode 100644 codecs/pressio/Cargo.toml create mode 120000 codecs/pressio/LICENSE create mode 100644 codecs/pressio/README.md create mode 100644 codecs/pressio/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 1608d699c..b329464b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ members = [ "codecs/linear-quantize", "codecs/log", "codecs/pco", + "codecs/pressio", "codecs/qpet-sperr", "codecs/random-projection", "codecs/reinterpret", @@ -69,6 +70,7 @@ numcodecs-lc = { version = "0.1", path = "codecs/lc", default-features = false } numcodecs-linear-quantize = { version = "0.5", path = "codecs/linear-quantize", default-features = false } numcodecs-log = { version = "0.5", path = "codecs/log", default-features = false } numcodecs-pco = { version = "0.3", path = "codecs/pco", default-features = false } +numcodecs-pressio = { version = "0.1", path = "codecs/pressio", default-features = false } numcodecs-qpet-sperr = { version = "0.2.2", path = "codecs/qpet-sperr", default-features = false } numcodecs-random-projection = { version = "0.4", path = "codecs/random-projection", default-features = false } numcodecs-reinterpret = { version = "0.4", path = "codecs/reinterpret", default-features = false } diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml new file mode 100644 index 000000000..2a8830c3f --- /dev/null +++ b/codecs/pressio/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "numcodecs-pressio" +version = "0.1.0" +edition = { workspace = true } +authors = { workspace = true } +repository = { workspace = true } +license = { workspace = true } +rust-version = { workspace = true } + +description = "libpressio codec wrapper for the numcodecs API" +readme = "README.md" +categories = ["compression", "encoding"] +keywords = ["libpressio", "numcodecs", "compression", "encoding"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +numcodecs = { workspace = true } +schemars = { workspace = true, features = ["derive", "preserve_order"] } +serde = { workspace = true, features = ["std", "derive"] } +thiserror = { workspace = true } + +# FIXME: move into workspace dependencies +libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "ddceba6" } + +[lints] +workspace = true diff --git a/codecs/pressio/LICENSE b/codecs/pressio/LICENSE new file mode 120000 index 000000000..30cff7403 --- /dev/null +++ b/codecs/pressio/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/codecs/pressio/README.md b/codecs/pressio/README.md new file mode 100644 index 000000000..2a211f673 --- /dev/null +++ b/codecs/pressio/README.md @@ -0,0 +1,38 @@ +[![CI Status]][workflow] [![MSRV]][repo] [![Latest Version]][crates.io] [![PyPi Release]][pypi] [![Rust Doc Crate]][docs.rs] [![Rust Doc Main]][docs] [![Read the Docs]][rtdocs] + +[CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main +[workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain + +[MSRV]: https://img.shields.io/badge/MSRV-1.85.0-blue +[repo]: https://github.com/juntyr/numcodecs-rs + +[Latest Version]: https://img.shields.io/crates/v/numcodecs-pressio +[crates.io]: https://crates.io/crates/numcodecs-pressio + +[PyPi Release]: https://img.shields.io/pypi/v/numcodecs-wasm-pressio.svg +[pypi]: https://pypi.python.org/pypi/numcodecs-wasm-pressio + +[Rust Doc Crate]: https://img.shields.io/docsrs/numcodecs-pressio +[docs.rs]: https://docs.rs/numcodecs-pressio/ + +[Rust Doc Main]: https://img.shields.io/badge/docs-main-blue +[docs]: https://juntyr.github.io/numcodecs-rs/numcodecs_pressio + +[Read the Docs]: https://img.shields.io/readthedocs/numcodecs-wasm?label=readthedocs +[rtdocs]: https://numcodecs-wasm.readthedocs.io/en/stable/api/numcodecs_wasm_pressio/ + +# numcodecs-pressio + +libpressio codec wrapper for the [`numcodecs`] API. + +[`numcodecs`]: https://docs.rs/numcodecs/0.2/numcodecs/ + +## License + +Licensed under the Mozilla Public License, Version 2.0 ([LICENSE](LICENSE) or https://www.mozilla.org/en-US/MPL/2.0/). + +## Funding + +The `numcodecs-pressio` crate has been developed as part of [ESiWACE3](https://www.esiwace.eu), the third phase of the Centre of Excellence in Simulation of Weather and Climate in Europe. + +Funded by the European Union. This work has received funding from the European High Performance Computing Joint Undertaking (JU) under grant agreement No 101093054. diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs new file mode 100644 index 000000000..cfe500cb9 --- /dev/null +++ b/codecs/pressio/src/lib.rs @@ -0,0 +1,82 @@ +//! [![CI Status]][workflow] [![MSRV]][repo] [![Latest Version]][crates.io] [![Rust Doc Crate]][docs.rs] [![Rust Doc Main]][docs] +//! +//! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main +//! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain +//! +//! [MSRV]: https://img.shields.io/badge/MSRV-1.85.0-blue +//! [repo]: https://github.com/juntyr/numcodecs-rs +//! +//! [Latest Version]: https://img.shields.io/crates/v/numcodecs-pressio +//! [crates.io]: https://crates.io/crates/numcodecs-pressio +//! +//! [Rust Doc Crate]: https://img.shields.io/docsrs/numcodecs-pressio +//! [docs.rs]: https://docs.rs/numcodecs-pressio/ +//! +//! [Rust Doc Main]: https://img.shields.io/badge/docs-main-blue +//! [docs]: https://juntyr.github.io/numcodecs-rs/numcodecs_pressio +//! +//! libpressio codec wrapper for the [`numcodecs`] API. + +use numcodecs::{ + AnyArray, AnyArrayAssignError, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, + StaticCodecConfig, StaticCodecVersion, +}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +/// Identity codec which applies the identity function, i.e. passes through the +/// input unchanged during encoding and decoding. +pub struct IdentityCodec { + /// The codec's encoding format version. Do not provide this parameter explicitly. + #[serde(default, rename = "_version")] + pub version: StaticCodecVersion<1, 0, 0>, +} + +impl Codec for IdentityCodec { + type Error = IdentityCodecError; + + fn encode(&self, data: AnyCowArray) -> Result { + Ok(data.into_owned()) + } + + fn decode(&self, encoded: AnyCowArray) -> Result { + Ok(encoded.into_owned()) + } + + fn decode_into( + &self, + encoded: AnyArrayView, + mut decoded: AnyArrayViewMut, + ) -> Result<(), Self::Error> { + Ok(decoded.assign(&encoded)?) + } +} + +impl StaticCodec for IdentityCodec { + const CODEC_ID: &'static str = "identity.rs"; + + type Config<'de> = Self; + + fn from_config(config: Self::Config<'_>) -> Self { + config + } + + fn get_config(&self) -> StaticCodecConfig { + StaticCodecConfig::from(self) + } +} + +#[derive(Debug, Error)] +/// Errors that may occur when applying the [`IdentityCodec`]. +pub enum IdentityCodecError { + /// [`IdentityCodec`] cannot decode into the provided array + #[error("Identity cannot decode into the provided array")] + MismatchedDecodeIntoArray { + /// The source of the error + #[from] + source: AnyArrayAssignError, + }, +} From f0b9581d23004845c8865ec42c910c25bd43307b Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Fri, 25 Apr 2025 07:50:21 +0000 Subject: [PATCH 02/55] Try with some libpressio fixes --- codecs/pressio/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index 2a8830c3f..bec78ed0a 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -21,7 +21,7 @@ serde = { workspace = true, features = ["std", "derive"] } thiserror = { workspace = true } # FIXME: move into workspace dependencies -libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "ddceba6" } +libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "82afbea" } [lints] workspace = true From d5a0800e4231cf1ce70fb107ed9b0786025df586 Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Fri, 25 Apr 2025 08:10:37 +0000 Subject: [PATCH 03/55] Skip numcodecs-pressio in non-builder WASM CI --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fb965be2..195cdbb63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,6 +93,7 @@ jobs: --exclude numcodecs-ebcc \ --exclude numcodecs-jpeg2000 \ --exclude numcodecs-lc \ + --exclude numcodecs-pressio \ --exclude numcodecs-qpet-sperr \ --exclude numcodecs-sperr \ --exclude numcodecs-sz3 \ @@ -229,6 +230,7 @@ jobs: --exclude numcodecs-ebcc \ --exclude numcodecs-jpeg2000 \ --exclude numcodecs-lc \ + --exclude numcodecs-pressio \ --exclude numcodecs-qpet-sperr \ --exclude numcodecs-sperr \ --exclude numcodecs-sz3 \ @@ -248,6 +250,7 @@ jobs: --exclude numcodecs-ebcc \ --exclude numcodecs-jpeg2000 \ --exclude numcodecs-lc \ + --exclude numcodecs-pressio \ --exclude numcodecs-qpet-sperr \ --exclude numcodecs-sperr \ --exclude numcodecs-sz3 \ From 1f7d1162a0f5f2d3f58f1d27a23921afd387e4af Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Fri, 25 Apr 2025 12:12:29 +0300 Subject: [PATCH 04/55] Update libpressio rev --- codecs/pressio/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index bec78ed0a..b3b7233ef 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -21,7 +21,7 @@ serde = { workspace = true, features = ["std", "derive"] } thiserror = { workspace = true } # FIXME: move into workspace dependencies -libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "82afbea" } +libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "6d25c06", default-features = false } [lints] workspace = true From 433f86bb88402072ca4d6b570fd642a941e0c98d Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Fri, 25 Apr 2025 12:29:15 +0300 Subject: [PATCH 05/55] Fix clippy lints --- codecs/pressio/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index cfe500cb9..4026c773d 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -17,6 +17,7 @@ //! //! libpressio codec wrapper for the [`numcodecs`] API. +use ::libpressio as _; use numcodecs::{ AnyArray, AnyArrayAssignError, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, StaticCodecConfig, StaticCodecVersion, From 9f380d19b20b3ab3cf592aba76cbb644520d82a8 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 12 Feb 2026 10:56:19 +0200 Subject: [PATCH 06/55] Some WASM hacks --- Cargo.toml | 1 + codecs/pressio/Cargo.toml | 4 +--- codecs/pressio/src/lib.rs | 22 +++++++++++----------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b329464b0..1213efb37 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,6 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "cad5bcc", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index b3b7233ef..d4cc6179a 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -15,13 +15,11 @@ keywords = ["libpressio", "numcodecs", "compression", "encoding"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +libpressio = { workspace = true } numcodecs = { workspace = true } schemars = { workspace = true, features = ["derive", "preserve_order"] } serde = { workspace = true, features = ["std", "derive"] } thiserror = { workspace = true } -# FIXME: move into workspace dependencies -libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "6d25c06", default-features = false } - [lints] workspace = true diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 4026c773d..814f054be 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -28,16 +28,16 @@ use thiserror::Error; #[derive(Clone, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] -/// Identity codec which applies the identity function, i.e. passes through the +/// Pressio codec which applies the identity function, i.e. passes through the /// input unchanged during encoding and decoding. -pub struct IdentityCodec { +pub struct PressioCodec { /// The codec's encoding format version. Do not provide this parameter explicitly. #[serde(default, rename = "_version")] pub version: StaticCodecVersion<1, 0, 0>, } -impl Codec for IdentityCodec { - type Error = IdentityCodecError; +impl Codec for PressioCodec { + type Error = PressioCodecError; fn encode(&self, data: AnyCowArray) -> Result { Ok(data.into_owned()) @@ -56,8 +56,8 @@ impl Codec for IdentityCodec { } } -impl StaticCodec for IdentityCodec { - const CODEC_ID: &'static str = "identity.rs"; +impl StaticCodec for PressioCodec { + const CODEC_ID: &'static str = "pressio.rs"; type Config<'de> = Self; @@ -65,16 +65,16 @@ impl StaticCodec for IdentityCodec { config } - fn get_config(&self) -> StaticCodecConfig { + fn get_config(&self) -> StaticCodecConfig<'_, Self> { StaticCodecConfig::from(self) } } #[derive(Debug, Error)] -/// Errors that may occur when applying the [`IdentityCodec`]. -pub enum IdentityCodecError { - /// [`IdentityCodec`] cannot decode into the provided array - #[error("Identity cannot decode into the provided array")] +/// Errors that may occur when applying the [`PressioCodec`]. +pub enum PressioCodecError { + /// [`PressioCodec`] cannot decode into the provided array + #[error("Pressio cannot decode into the provided array")] MismatchedDecodeIntoArray { /// The source of the error #[from] From beb5952180beb216088db25408f90cca3696c95b Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 12 Feb 2026 11:03:24 +0200 Subject: [PATCH 07/55] Clean up libpressio dependencies --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 1213efb37..fd9773327 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "cad5bcc", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "3a00062", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From ffd2a58340852007a5681a6121bd0e8eafc606d2 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 12 Feb 2026 12:25:26 +0200 Subject: [PATCH 08/55] Some experimentation to produce more link errors --- codecs/pressio/src/lib.rs | 183 ++++++++++++++++++++++++++++++++++---- 1 file changed, 164 insertions(+), 19 deletions(-) diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 814f054be..d35129594 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -17,42 +17,191 @@ //! //! libpressio codec wrapper for the [`numcodecs`] API. -use ::libpressio as _; +use std::{borrow::Cow, collections::BTreeMap, sync::LazyLock}; + use numcodecs::{ - AnyArray, AnyArrayAssignError, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, - StaticCodecConfig, StaticCodecVersion, + AnyArray, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, StaticCodecConfig, + StaticCodecVersion, }; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; +use schemars::{JsonSchema, Schema, SchemaGenerator}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; +static PRESSIO: LazyLock = LazyLock::new(Pressio::new); + #[derive(Clone, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] /// Pressio codec which applies the identity function, i.e. passes through the /// input unchanged during encoding and decoding. pub struct PressioCodec { + /// The Pressio compressor + #[serde(flatten)] + pub compressor: PressioCompressor, /// The codec's encoding format version. Do not provide this parameter explicitly. #[serde(default, rename = "_version")] pub version: StaticCodecVersion<1, 0, 0>, } +/// Pressio compressor +pub struct PressioCompressor { + format: PressioCompressorFormat, + compressor: libpressio::PressioCompressor, +} + +// FIXME: UNSOUND +#[expect(unsafe_code, clippy::non_send_fields_in_send_ty)] +unsafe impl Send for PressioCompressor {} +#[expect(unsafe_code)] +unsafe impl Sync for PressioCompressor {} + +impl Clone for PressioCompressor { + #[expect(clippy::unwrap_used)] + fn clone(&self) -> Self { + let pressio = PRESSIO.get_or_unwrap(); + let compressor = pressio.get_compressor(self.format.id.as_str()).unwrap(); + let options = self.compressor.get_options().unwrap(); + compressor.set_options(&options).unwrap(); + + Self { + format: self.format.clone(), + compressor, + } + } +} + +impl Serialize for PressioCompressor { + fn serialize(&self, serializer: S) -> Result { + self.format.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for PressioCompressor { + fn deserialize>(deserializer: D) -> Result { + let pressio = PRESSIO + .get() + .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + + // TODO: better error handling + let format = PressioCompressorFormat::deserialize(deserializer)?; + + let compressor = pressio + .get_compressor(format.id.as_str()) + .map_err(|err| serde::de::Error::custom(err.message))?; + let mut options = compressor + .get_options() + .map_err(|err| serde::de::Error::custom(err.message))?; + + for (key, value) in &format.options { + options = options + .set( + key, + match value { + PressioOption::U8(x) => libpressio::PressioOption::uint8(Some(*x)), + PressioOption::I8(x) => libpressio::PressioOption::int8(Some(*x)), + PressioOption::U16(x) => libpressio::PressioOption::uint16(Some(*x)), + PressioOption::I16(x) => libpressio::PressioOption::int16(Some(*x)), + PressioOption::U32(x) => libpressio::PressioOption::uint32(Some(*x)), + PressioOption::I32(x) => libpressio::PressioOption::int32(Some(*x)), + PressioOption::U64(x) => libpressio::PressioOption::uint64(Some(*x)), + PressioOption::I64(x) => libpressio::PressioOption::int64(Some(*x)), + PressioOption::F32(x) => libpressio::PressioOption::float32(Some(*x)), + PressioOption::F64(x) => libpressio::PressioOption::float64(Some(*x)), + PressioOption::String(x) => { + libpressio::PressioOption::string(Some(x.clone())) + } + PressioOption::VecString(x) => { + libpressio::PressioOption::vec_string(Some(x.clone())) + } + }, + ) + .map_err(|err| serde::de::Error::custom(err.message))?; + } + + Ok(Self { format, compressor }) + } +} + +impl JsonSchema for PressioCompressor { + fn schema_name() -> Cow<'static, str> { + PressioCompressorFormat::schema_name() + } + + fn json_schema(generator: &mut SchemaGenerator) -> Schema { + PressioCompressorFormat::json_schema(generator) + } +} + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename = "PressioCompressor")] +struct PressioCompressorFormat { + id: String, + #[serde(flatten)] + options: BTreeMap, +} + +#[expect(missing_docs)] +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(untagged)] +/// Pressio option value +pub enum PressioOption { + U8(u8), + I8(i8), + U16(u16), + I16(i16), + U32(u32), + I32(i32), + U64(u64), + I64(i64), + F32(f32), + F64(f64), + String(String), + VecString(Vec), +} + +struct Pressio { + pressio: Result, +} + +impl Pressio { + fn new() -> Self { + Self { + pressio: libpressio::Pressio::new(), + } + } + + const fn get(&self) -> Result<&libpressio::Pressio, &libpressio::PressioError> { + self.pressio.as_ref() + } + + #[expect(clippy::unwrap_used)] + fn get_or_unwrap(&self) -> &libpressio::Pressio { + self.pressio.as_ref().unwrap() + } +} + +// FIXME: UNSOUND +#[expect(unsafe_code, clippy::non_send_fields_in_send_ty)] +unsafe impl Send for Pressio {} +#[expect(unsafe_code)] +unsafe impl Sync for Pressio {} + impl Codec for PressioCodec { type Error = PressioCodecError; - fn encode(&self, data: AnyCowArray) -> Result { - Ok(data.into_owned()) + fn encode(&self, _data: AnyCowArray) -> Result { + Err(PressioCodecError::Unimplemented) } - fn decode(&self, encoded: AnyCowArray) -> Result { - Ok(encoded.into_owned()) + fn decode(&self, _encoded: AnyCowArray) -> Result { + Err(PressioCodecError::Unimplemented) } fn decode_into( &self, - encoded: AnyArrayView, - mut decoded: AnyArrayViewMut, + _encoded: AnyArrayView, + _decoded: AnyArrayViewMut, ) -> Result<(), Self::Error> { - Ok(decoded.assign(&encoded)?) + Err(PressioCodecError::Unimplemented) } } @@ -73,11 +222,7 @@ impl StaticCodec for PressioCodec { #[derive(Debug, Error)] /// Errors that may occur when applying the [`PressioCodec`]. pub enum PressioCodecError { - /// [`PressioCodec`] cannot decode into the provided array - #[error("Pressio cannot decode into the provided array")] - MismatchedDecodeIntoArray { - /// The source of the error - #[from] - source: AnyArrayAssignError, - }, + /// [`PressioCodec`] does not yet implement this functionality + #[error("Pressio does not yet implement this functionality")] + Unimplemented, } From 2890ed02d457e8a4b21920dd560f86b19c87339e Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 12 Feb 2026 19:03:37 +0200 Subject: [PATCH 09/55] Upgrade to upstream std_compat --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index fd9773327..56cc6eeb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "3a00062", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5d91490", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From 448382130b369b30ca00ccd906edad0ccfe7fe7e Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 12 Feb 2026 21:19:54 +0200 Subject: [PATCH 10/55] Only link libstdc++ in libpressio on Linux target --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 56cc6eeb0..bb4f9040b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5d91490", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "818f30d", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index d35129594..e69c734b5 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -226,3 +226,8 @@ pub enum PressioCodecError { #[error("Pressio does not yet implement this functionality")] Unimplemented, } + +// FIXME: don't stub +#[expect(unsafe_code)] +#[unsafe(no_mangle)] +const extern "C" fn pressio_register_all() {} From 27d17e863aa2e54ee9807df4952e5ef7bcc8f928 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 12 Feb 2026 22:15:54 +0200 Subject: [PATCH 11/55] Fix target_os check in libpressio build.rs --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index bb4f9040b..dd71b85c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "818f30d", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "a1f99af", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From 4ce862c6b39e1f66d337c3b44723375494d89cdf Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Tue, 17 Feb 2026 09:50:48 +0200 Subject: [PATCH 12/55] no really really don't ask --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dd71b85c5..2ab2760c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "a1f99af", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "a664de1", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index e69c734b5..d35129594 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -226,8 +226,3 @@ pub enum PressioCodecError { #[error("Pressio does not yet implement this functionality")] Unimplemented, } - -// FIXME: don't stub -#[expect(unsafe_code)] -#[unsafe(no_mangle)] -const extern "C" fn pressio_register_all() {} From de373d8dbf1b3e53b2f67898a11e92a28cc9c89f Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Tue, 17 Feb 2026 14:19:26 +0200 Subject: [PATCH 13/55] add some option parsing and inspection support --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 67 +++++++++++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2ab2760c9..ff268c4ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "a664de1", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "435c82a", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index d35129594..40bcbbeda 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -30,7 +30,7 @@ use thiserror::Error; static PRESSIO: LazyLock = LazyLock::new(Pressio::new); #[derive(Clone, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] +#[schemars(deny_unknown_fields)] /// Pressio codec which applies the identity function, i.e. passes through the /// input unchanged during encoding and decoding. pub struct PressioCodec { @@ -58,7 +58,9 @@ impl Clone for PressioCompressor { #[expect(clippy::unwrap_used)] fn clone(&self) -> Self { let pressio = PRESSIO.get_or_unwrap(); - let compressor = pressio.get_compressor(self.format.id.as_str()).unwrap(); + let compressor = pressio + .get_compressor(self.format.compressor.as_str()) + .unwrap(); let options = self.compressor.get_options().unwrap(); compressor.set_options(&options).unwrap(); @@ -80,13 +82,26 @@ impl<'de> Deserialize<'de> for PressioCompressor { let pressio = PRESSIO .get() .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; - // TODO: better error handling let format = PressioCompressorFormat::deserialize(deserializer)?; - let compressor = pressio - .get_compressor(format.id.as_str()) - .map_err(|err| serde::de::Error::custom(err.message))?; + .get_compressor(format.compressor.as_str()) + .map_err(|err| { + let supported_compressors = + pressio + .supported_compressors() + .map_or(String::from(""), |x| { + x.iter() + .map(|x| format!("`{x}`")) + .collect::>() + .join(", ") + }); + + serde::de::Error::custom(format_args!( + "{}, choose one of: {}", + err.message, supported_compressors + )) + })?; let mut options = compressor .get_options() .map_err(|err| serde::de::Error::custom(err.message))?; @@ -96,6 +111,7 @@ impl<'de> Deserialize<'de> for PressioCompressor { .set( key, match value { + PressioOption::Bool(x) => libpressio::PressioOption::bool(Some(*x)), PressioOption::U8(x) => libpressio::PressioOption::uint8(Some(*x)), PressioOption::I8(x) => libpressio::PressioOption::int8(Some(*x)), PressioOption::U16(x) => libpressio::PressioOption::uint16(Some(*x)), @@ -117,6 +133,35 @@ impl<'de> Deserialize<'de> for PressioCompressor { .map_err(|err| serde::de::Error::custom(err.message))?; } + let mut format = format; + if let Ok(format_options) = options.get_options() { + format.options = format_options + .into_iter() + .filter_map(|(k, v)| match v { + libpressio::PressioOption::bool(Some(x)) => Some((k, PressioOption::Bool(x))), + libpressio::PressioOption::int8(Some(x)) => Some((k, PressioOption::I8(x))), + libpressio::PressioOption::int16(Some(x)) => Some((k, PressioOption::I16(x))), + libpressio::PressioOption::int32(Some(x)) => Some((k, PressioOption::I32(x))), + libpressio::PressioOption::int64(Some(x)) => Some((k, PressioOption::I64(x))), + libpressio::PressioOption::uint8(Some(x)) => Some((k, PressioOption::U8(x))), + libpressio::PressioOption::uint16(Some(x)) => Some((k, PressioOption::U16(x))), + libpressio::PressioOption::uint32(Some(x)) => Some((k, PressioOption::U32(x))), + libpressio::PressioOption::uint64(Some(x)) => Some((k, PressioOption::U64(x))), + libpressio::PressioOption::float32(Some(x)) => Some((k, PressioOption::F32(x))), + libpressio::PressioOption::float64(Some(x)) => Some((k, PressioOption::F64(x))), + libpressio::PressioOption::string(Some(x)) => { + Some((k, PressioOption::String(x))) + } + // FIXME: seems to return strings as a single joined string + libpressio::PressioOption::vec_string(Some(x)) => { + Some((k, PressioOption::VecString(x))) + } + _ => None, + }) + .collect(); + } + let format = format; + Ok(Self { format, compressor }) } } @@ -131,19 +176,21 @@ impl JsonSchema for PressioCompressor { } } -#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(rename = "PressioCompressor")] struct PressioCompressorFormat { - id: String, - #[serde(flatten)] + compressor: String, + // TODO: flatten + #[serde(default)] options: BTreeMap, } #[expect(missing_docs)] -#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(untagged)] /// Pressio option value pub enum PressioOption { + Bool(bool), U8(u8), I8(i8), U16(u16), From 452704f37105a429ef38385bd55e2f0593b34eae Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 18 Feb 2026 08:43:40 +0200 Subject: [PATCH 14/55] some more plugins --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ff268c4ef..297eddf6d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "435c82a", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "e562964", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From 4d60a2f9acc08680769f01d3972350b0d448c12f Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 18 Feb 2026 15:04:06 +0200 Subject: [PATCH 15/55] enable more plugins --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 297eddf6d..eea5f15ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "e562964", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "f142487", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From eb0fd9049771c668e0bddf0f47684caa2a5a0e1b Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sat, 21 Feb 2026 06:36:13 +0200 Subject: [PATCH 16/55] Simple encode/decode implementation --- Cargo.toml | 2 +- codecs/pressio/Cargo.toml | 1 + codecs/pressio/src/lib.rs | 167 +++++++++++++++++++++++++++++++++++--- 3 files changed, 157 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index eea5f15ad..508f8cb5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "f142487", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "4e7b44f", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index d4cc6179a..e664605e6 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -16,6 +16,7 @@ keywords = ["libpressio", "numcodecs", "compression", "encoding"] [dependencies] libpressio = { workspace = true } +ndarray = { workspace = true } numcodecs = { workspace = true } schemars = { workspace = true, features = ["derive", "preserve_order"] } serde = { workspace = true, features = ["std", "derive"] } diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 40bcbbeda..56fd0f3ea 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -19,9 +19,10 @@ use std::{borrow::Cow, collections::BTreeMap, sync::LazyLock}; +use ndarray::{CowArray, IxDyn}; use numcodecs::{ - AnyArray, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, StaticCodecConfig, - StaticCodecVersion, + AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray, + Codec, StaticCodec, StaticCodecConfig, StaticCodecVersion, }; use schemars::{JsonSchema, Schema, SchemaGenerator}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -235,20 +236,126 @@ unsafe impl Sync for Pressio {} impl Codec for PressioCodec { type Error = PressioCodecError; - fn encode(&self, _data: AnyCowArray) -> Result { - Err(PressioCodecError::Unimplemented) + fn encode(&self, data: AnyCowArray) -> Result { + fn encode_typed( + compressor: &libpressio::PressioCompressor, + data: CowArray, + ) -> Result { + let data = match data.try_into_owned_nocopy() { + Ok(data) => libpressio::PressioData::new(data), + Err(data) => libpressio::PressioData::new_copied(data.view()), + }; + + let compressed_data = + libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); + + let compressed_data = compressor.compress(&data, compressed_data).map_err(|err| { + PressioCodecError::PressioEncodeFailed { + source: PressioCodingError(err), + } + })?; + + let Some(compressed_data) = compressed_data.clone_into_array() else { + return Err(PressioCodecError::EncodeToUnknownDtype); + }; + + match compressed_data { + libpressio::PressioArray::Bool(_) => Err(PressioCodecError::EncodeToBoolArray), + libpressio::PressioArray::U8(a) | libpressio::PressioArray::Byte(a) => { + Ok(AnyArray::U8(a)) + } + libpressio::PressioArray::U16(a) => Ok(AnyArray::U16(a)), + libpressio::PressioArray::U32(a) => Ok(AnyArray::U32(a)), + libpressio::PressioArray::U64(a) => Ok(AnyArray::U64(a)), + libpressio::PressioArray::I8(a) => Ok(AnyArray::I8(a)), + libpressio::PressioArray::I16(a) => Ok(AnyArray::I16(a)), + libpressio::PressioArray::I32(a) => Ok(AnyArray::I32(a)), + libpressio::PressioArray::I64(a) => Ok(AnyArray::I64(a)), + libpressio::PressioArray::F32(a) => Ok(AnyArray::F32(a)), + libpressio::PressioArray::F64(a) => Ok(AnyArray::F64(a)), + } + } + + match data { + AnyCowArray::U8(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::U16(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::U32(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::U64(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::I8(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::I16(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::I32(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::I64(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::F32(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::F64(data) => encode_typed(&self.compressor.compressor, data), + data => Err(PressioCodecError::UnsupportedDtype(data.dtype())), + } } - fn decode(&self, _encoded: AnyCowArray) -> Result { - Err(PressioCodecError::Unimplemented) + fn decode(&self, encoded: AnyCowArray) -> Result { + fn decode_typed( + compressor: &libpressio::PressioCompressor, + encoded: CowArray, + ) -> Result { + let encoded = match encoded.try_into_owned_nocopy() { + Ok(encoded) => libpressio::PressioData::new(encoded), + Err(encoded) => libpressio::PressioData::new_copied(encoded.view()), + }; + + let decompressed_data = + libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); + + let decompressed_data = + compressor + .compress(&encoded, decompressed_data) + .map_err(|err| PressioCodecError::PressioDecodeFailed { + source: PressioCodingError(err), + })?; + + let Some(decompressed_data) = decompressed_data.clone_into_array() else { + return Err(PressioCodecError::DecodeToUnknownDtype); + }; + + match decompressed_data { + libpressio::PressioArray::Bool(_) => Err(PressioCodecError::DecodeToBoolArray), + libpressio::PressioArray::U8(a) | libpressio::PressioArray::Byte(a) => { + Ok(AnyArray::U8(a)) + } + libpressio::PressioArray::U16(a) => Ok(AnyArray::U16(a)), + libpressio::PressioArray::U32(a) => Ok(AnyArray::U32(a)), + libpressio::PressioArray::U64(a) => Ok(AnyArray::U64(a)), + libpressio::PressioArray::I8(a) => Ok(AnyArray::I8(a)), + libpressio::PressioArray::I16(a) => Ok(AnyArray::I16(a)), + libpressio::PressioArray::I32(a) => Ok(AnyArray::I32(a)), + libpressio::PressioArray::I64(a) => Ok(AnyArray::I64(a)), + libpressio::PressioArray::F32(a) => Ok(AnyArray::F32(a)), + libpressio::PressioArray::F64(a) => Ok(AnyArray::F64(a)), + } + } + + match encoded { + AnyCowArray::U8(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::U16(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::U32(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::U64(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::I8(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::I16(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::I32(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::I64(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::F32(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::F64(encoded) => decode_typed(&self.compressor.compressor, encoded), + encoded => Err(PressioCodecError::UnsupportedDtype(encoded.dtype())), + } } fn decode_into( &self, - _encoded: AnyArrayView, - _decoded: AnyArrayViewMut, + encoded: AnyArrayView, + mut decoded: AnyArrayViewMut, ) -> Result<(), Self::Error> { - Err(PressioCodecError::Unimplemented) + // TODO: optimize + let decoded_in = self.decode(encoded.cow())?; + + Ok(decoded.assign(&decoded_in)?) } } @@ -269,7 +376,43 @@ impl StaticCodec for PressioCodec { #[derive(Debug, Error)] /// Errors that may occur when applying the [`PressioCodec`]. pub enum PressioCodecError { - /// [`PressioCodec`] does not yet implement this functionality - #[error("Pressio does not yet implement this functionality")] - Unimplemented, + /// [`PressioCodec`] does not support the dtype + #[error("Pressio does not support the dtype {0}")] + UnsupportedDtype(AnyArrayDType), + /// [`PressioCodec`] failed to encode the data + #[error("Pressio failed to encode the data")] + PressioEncodeFailed { + /// Opaque source error + source: PressioCodingError, + }, + /// [`PressioCodec`] encoded to an unknown unsupported dtype + #[error("Pressio encoded to an unknown unsupported dtype")] + EncodeToUnknownDtype, + /// [`PressioCodec`] encoded to a bool array, which is unsupported + #[error("Pressio encoded to a bool array, which is unsupported")] + EncodeToBoolArray, + /// [`PressioCodec`] failed to decode the data + #[error("Pressio failed to decode the data")] + PressioDecodeFailed { + /// Opaque source error + source: PressioCodingError, + }, + /// [`PressioCodec`] decoded to an unknown unsupported dtype + #[error("Pressio decoded to an unknown unsupported dtype")] + DecodeToUnknownDtype, + /// [`PressioCodec`] decoded to a bool array, which is unsupported + #[error("Pressio decoded to a bool array, which is unsupported")] + DecodeToBoolArray, + /// [`PressioCodec`] cannot decode into the provided array + #[error("Pressio cannot decode into the provided array")] + MismatchedDecodeIntoArray { + /// The source of the error + #[from] + source: AnyArrayAssignError, + }, } + +#[derive(Debug, Error)] +#[error(transparent)] +/// Opaque error for when encoding or decoding with libpressio fails +pub struct PressioCodingError(libpressio::PressioError); From 5ff284b50029a38f5e18e26ee4b45f7805f33b18 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 23 Feb 2026 12:37:52 +0200 Subject: [PATCH 17/55] add host libcxx to the nix flake --- Cargo.toml | 2 +- .../numcodecs-wasm-builder/buildenv/flake.nix | 2 ++ crates/numcodecs-wasm-builder/src/main.rs | 20 +++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 508f8cb5c..56d45c501 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "4e7b44f", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "3b90d94", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/crates/numcodecs-wasm-builder/buildenv/flake.nix b/crates/numcodecs-wasm-builder/buildenv/flake.nix index b02efb029..3e4b8bab1 100644 --- a/crates/numcodecs-wasm-builder/buildenv/flake.nix +++ b/crates/numcodecs-wasm-builder/buildenv/flake.nix @@ -61,6 +61,7 @@ packages = [ (pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain) pkgs."llvmPackages_${llvmVersion}".libclang + pkgs."llvmPackages_${llvmVersion}".libcxx wasi-sysroot libclang_rt pkgs.cmake @@ -84,6 +85,7 @@ MY_WASM_OPT = "${pkgs.binaryen}/bin/wasm-opt"; MY_PKG_CONFIG = "${pkgs.pkg-config}/bin/pkg-config"; MY_PYTHON3 = "${pkgs.python3}/bin/python3"; + MY_HOST_LIBCXX = "${pkgs."llvmPackages_${llvmVersion}".libcxx}"; }; }; }); diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index 329d6acf0..577da25c0 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -203,6 +203,8 @@ struct NixEnv { pkg_config: PathBuf, #[expect(dead_code)] python3: PathBuf, + host_libcxx: PathBuf, + host_sysroot: PathBuf, } impl NixEnv { @@ -270,6 +272,11 @@ impl NixEnv { wasm_opt: try_read_env(&env, "MY_WASM_OPT")?, pkg_config: try_read_env(&env, "MY_PKG_CONFIG")?, python3: try_read_env(&env, "MY_PYTHON3")?, + host_libcxx: try_read_env(&env, "MY_HOST_LIBCXX")?, + // FIXME + host_sysroot: PathBuf::from( + "/nix/store/5gfsv5n8zhpnl9yhggjpxrxg0jyflwja-apple-sdk-11.3/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk", + ), }) } } @@ -295,6 +302,8 @@ fn configure_cargo_cmd( wasi_sysroot, libclang_rt, pkg_config, + host_libcxx, + host_sysroot, .. } = nix_env; @@ -379,6 +388,17 @@ fn configure_cargo_cmd( cpp_include_path = crate_dir.join("include.hpp").display(), debug = if debug { "-g" } else { "" }, )); + cmd.arg(format!( + "CXXFLAGSHOST=-isysroot {host_sysroot} -isystem {host_libcxx_include} \ + -isystem {clang_include}", + host_sysroot = host_sysroot.display(), + host_libcxx_include = host_libcxx.join("include").join("c++").join("v1").display(), + clang_include = libclang + .join("clang") + .join(llvm_version) + .join("include") + .display(), + )); cmd.arg(format!( "BINDGEN_EXTRA_CLANG_ARGS=--target=wasm32-wasip1 -nodefaultlibs -resource-dir \ {resource_dir} --sysroot={wasi_sysroot} -isystem {wasm32_wasi_cxx_include} -isystem \ From 00126c4e9903a5a352786e623e13e33666aa8e38 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 23 Feb 2026 13:04:04 +0200 Subject: [PATCH 18/55] fix cmake env lookup --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 56d45c501..88f0202b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "3b90d94", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5bc8ff0", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From d45cbae68f602dc69d719339b0108a7c9da6d184 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 23 Feb 2026 14:52:10 +0200 Subject: [PATCH 19/55] Fix clippy lint --- codecs/pressio/src/lib.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 56fd0f3ea..ad5a58597 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -88,15 +88,15 @@ impl<'de> Deserialize<'de> for PressioCompressor { let compressor = pressio .get_compressor(format.compressor.as_str()) .map_err(|err| { - let supported_compressors = - pressio - .supported_compressors() - .map_or(String::from(""), |x| { - x.iter() - .map(|x| format!("`{x}`")) - .collect::>() - .join(", ") - }); + let supported_compressors = pressio.supported_compressors().map_or_else( + |_| String::from(""), + |x| { + x.iter() + .map(|x| format!("`{x}`")) + .collect::>() + .join(", ") + }, + ); serde::de::Error::custom(format_args!( "{}, choose one of: {}", From 392f9cd2a00cd8151eb48fb37598005acf34b898 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sun, 1 Mar 2026 23:08:36 +0200 Subject: [PATCH 20/55] Pressio and PressioCompressor are Send + !Sync --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 148 ++++++++++++++++++++------------------ 2 files changed, 80 insertions(+), 70 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 88f0202b5..3d31188ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5bc8ff0", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "fea47ef", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index ad5a58597..40b008b06 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -17,7 +17,11 @@ //! //! libpressio codec wrapper for the [`numcodecs`] API. -use std::{borrow::Cow, collections::BTreeMap, sync::LazyLock}; +use std::{ + borrow::Cow, + collections::BTreeMap, + sync::{LazyLock, Mutex}, +}; use ndarray::{CowArray, IxDyn}; use numcodecs::{ @@ -46,28 +50,24 @@ pub struct PressioCodec { /// Pressio compressor pub struct PressioCompressor { format: PressioCompressorFormat, - compressor: libpressio::PressioCompressor, + compressor: Mutex, } -// FIXME: UNSOUND -#[expect(unsafe_code, clippy::non_send_fields_in_send_ty)] -unsafe impl Send for PressioCompressor {} -#[expect(unsafe_code)] -unsafe impl Sync for PressioCompressor {} - impl Clone for PressioCompressor { #[expect(clippy::unwrap_used)] fn clone(&self) -> Self { - let pressio = PRESSIO.get_or_unwrap(); - let compressor = pressio - .get_compressor(self.format.compressor.as_str()) - .unwrap(); - let options = self.compressor.get_options().unwrap(); + let mut compressor = { + let mut pressio = PRESSIO.get_or_unwrap().lock().unwrap(); + pressio + .get_compressor(self.format.compressor.as_str()) + .unwrap() + }; + let options = self.compressor.lock().unwrap().get_options().unwrap(); compressor.set_options(&options).unwrap(); Self { format: self.format.clone(), - compressor, + compressor: Mutex::new(compressor), } } } @@ -80,29 +80,32 @@ impl Serialize for PressioCompressor { impl<'de> Deserialize<'de> for PressioCompressor { fn deserialize>(deserializer: D) -> Result { - let pressio = PRESSIO - .get() - .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; // TODO: better error handling let format = PressioCompressorFormat::deserialize(deserializer)?; - let compressor = pressio - .get_compressor(format.compressor.as_str()) - .map_err(|err| { - let supported_compressors = pressio.supported_compressors().map_or_else( - |_| String::from(""), - |x| { - x.iter() - .map(|x| format!("`{x}`")) - .collect::>() - .join(", ") - }, - ); - - serde::de::Error::custom(format_args!( - "{}, choose one of: {}", - err.message, supported_compressors - )) - })?; + let compressor = { + let pressio = PRESSIO + .get() + .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + let mut pressio = pressio.lock().map_err(serde::de::Error::custom)?; + pressio + .get_compressor(format.compressor.as_str()) + .map_err(|err| { + let supported_compressors = pressio.supported_compressors().map_or_else( + |_| String::from(""), + |x| { + x.iter() + .map(|x| format!("`{x}`")) + .collect::>() + .join(", ") + }, + ); + + serde::de::Error::custom(format_args!( + "{}, choose one of: {}", + err.message, supported_compressors + )) + })? + }; let mut options = compressor .get_options() .map_err(|err| serde::de::Error::custom(err.message))?; @@ -161,9 +164,11 @@ impl<'de> Deserialize<'de> for PressioCompressor { }) .collect(); } - let format = format; - Ok(Self { format, compressor }) + Ok(Self { + format, + compressor: Mutex::new(compressor), + }) } } @@ -207,38 +212,32 @@ pub enum PressioOption { } struct Pressio { - pressio: Result, + pressio: Result, libpressio::PressioError>, } impl Pressio { fn new() -> Self { Self { - pressio: libpressio::Pressio::new(), + pressio: libpressio::Pressio::new().map(Mutex::new), } } - const fn get(&self) -> Result<&libpressio::Pressio, &libpressio::PressioError> { + const fn get(&self) -> Result<&Mutex, &libpressio::PressioError> { self.pressio.as_ref() } #[expect(clippy::unwrap_used)] - fn get_or_unwrap(&self) -> &libpressio::Pressio { + fn get_or_unwrap(&self) -> &Mutex { self.pressio.as_ref().unwrap() } } -// FIXME: UNSOUND -#[expect(unsafe_code, clippy::non_send_fields_in_send_ty)] -unsafe impl Send for Pressio {} -#[expect(unsafe_code)] -unsafe impl Sync for Pressio {} - impl Codec for PressioCodec { type Error = PressioCodecError; fn encode(&self, data: AnyCowArray) -> Result { fn encode_typed( - compressor: &libpressio::PressioCompressor, + compressor: &mut libpressio::PressioCompressor, data: CowArray, ) -> Result { let data = match data.try_into_owned_nocopy() { @@ -276,24 +275,28 @@ impl Codec for PressioCodec { } } + let Ok(mut compressor) = self.compressor.compressor.lock() else { + return Err(PressioCodecError::PressioPoisonedMutex); + }; + match data { - AnyCowArray::U8(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::U16(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::U32(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::U64(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::I8(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::I16(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::I32(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::I64(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::F32(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::F64(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::U8(data) => encode_typed(&mut compressor, data), + AnyCowArray::U16(data) => encode_typed(&mut compressor, data), + AnyCowArray::U32(data) => encode_typed(&mut compressor, data), + AnyCowArray::U64(data) => encode_typed(&mut compressor, data), + AnyCowArray::I8(data) => encode_typed(&mut compressor, data), + AnyCowArray::I16(data) => encode_typed(&mut compressor, data), + AnyCowArray::I32(data) => encode_typed(&mut compressor, data), + AnyCowArray::I64(data) => encode_typed(&mut compressor, data), + AnyCowArray::F32(data) => encode_typed(&mut compressor, data), + AnyCowArray::F64(data) => encode_typed(&mut compressor, data), data => Err(PressioCodecError::UnsupportedDtype(data.dtype())), } } fn decode(&self, encoded: AnyCowArray) -> Result { fn decode_typed( - compressor: &libpressio::PressioCompressor, + compressor: &mut libpressio::PressioCompressor, encoded: CowArray, ) -> Result { let encoded = match encoded.try_into_owned_nocopy() { @@ -332,17 +335,21 @@ impl Codec for PressioCodec { } } + let Ok(mut compressor) = self.compressor.compressor.lock() else { + return Err(PressioCodecError::PressioPoisonedMutex); + }; + match encoded { - AnyCowArray::U8(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::U16(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::U32(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::U64(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::I8(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::I16(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::I32(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::I64(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::F32(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::F64(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::U8(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::U16(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::U32(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::U64(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::I8(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::I16(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::I32(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::I64(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::F32(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::F64(encoded) => decode_typed(&mut compressor, encoded), encoded => Err(PressioCodecError::UnsupportedDtype(encoded.dtype())), } } @@ -379,6 +386,9 @@ pub enum PressioCodecError { /// [`PressioCodec`] does not support the dtype #[error("Pressio does not support the dtype {0}")] UnsupportedDtype(AnyArrayDType), + /// [`PressioCodec`] lock was poisoned + #[error("Pressio lock was poisoned")] + PressioPoisonedMutex, /// [`PressioCodec`] failed to encode the data #[error("Pressio failed to encode the data")] PressioEncodeFailed { From b67439ec21862c906e941a595d4f71179cbc0543 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 2 Mar 2026 10:03:15 +0200 Subject: [PATCH 21/55] derive host sysroot in builder from clang include paths --- Cargo.toml | 2 +- .../numcodecs-wasm-builder/buildenv/flake.nix | 2 +- crates/numcodecs-wasm-builder/src/main.rs | 68 ++++++++++++++++--- 3 files changed, 61 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3d31188ad..666a5f759 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "fea47ef", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "6329913", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/crates/numcodecs-wasm-builder/buildenv/flake.nix b/crates/numcodecs-wasm-builder/buildenv/flake.nix index 3e4b8bab1..22ce9af96 100644 --- a/crates/numcodecs-wasm-builder/buildenv/flake.nix +++ b/crates/numcodecs-wasm-builder/buildenv/flake.nix @@ -85,7 +85,7 @@ MY_WASM_OPT = "${pkgs.binaryen}/bin/wasm-opt"; MY_PKG_CONFIG = "${pkgs.pkg-config}/bin/pkg-config"; MY_PYTHON3 = "${pkgs.python3}/bin/python3"; - MY_HOST_LIBCXX = "${pkgs."llvmPackages_${llvmVersion}".libcxx}"; + MY_HOST_LIBCXX = "${pkgs."llvmPackages_${llvmVersion}".libcxx.dev}"; }; }; }); diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index 577da25c0..7fbb48295 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -3,9 +3,12 @@ use std::{ collections::HashMap, - env, fs, io, + env, + ffi::OsStr, + fs, io, + os::unix::ffi::OsStrExt, path::{Path, PathBuf}, - process::Command, + process::{Command, Stdio}, str::FromStr, }; @@ -69,9 +72,11 @@ fn main() -> io::Result<()> { copy_buildenv_to_crate(&crate_dir)?; let nix_env = NixEnv::new(&crate_dir)?; + let host_sysroot = find_clang_host_sysroot(&nix_env, &crate_dir)?; let wasm = build_wasm_codec( &nix_env, + &host_sysroot, &target_dir, &crate_dir, &format!("{}-wasm", args.crate_), @@ -204,7 +209,6 @@ struct NixEnv { #[expect(dead_code)] python3: PathBuf, host_libcxx: PathBuf, - host_sysroot: PathBuf, } impl NixEnv { @@ -273,17 +277,63 @@ impl NixEnv { pkg_config: try_read_env(&env, "MY_PKG_CONFIG")?, python3: try_read_env(&env, "MY_PYTHON3")?, host_libcxx: try_read_env(&env, "MY_HOST_LIBCXX")?, - // FIXME - host_sysroot: PathBuf::from( - "/nix/store/5gfsv5n8zhpnl9yhggjpxrxg0jyflwja-apple-sdk-11.3/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk", - ), }) } } +fn find_clang_host_sysroot(nix_env: &NixEnv, flake_parent_dir: &Path) -> io::Result { + let NixEnv { clang, .. } = nix_env; + + let mut cmd = Command::new("nix"); + cmd.current_dir(flake_parent_dir); + cmd.arg("develop"); + // cmd.arg("--store"); + // cmd.arg(nix_store_path); + cmd.arg("--no-update-lock-file"); + cmd.arg("--ignore-environment"); + cmd.arg("path:."); + cmd.arg("--command"); + cmd.arg(clang.join("clang")); + cmd.arg("-v"); + cmd.arg("-x"); + cmd.arg("c"); + cmd.arg("-c"); + cmd.arg("-"); + cmd.stdin(Stdio::null()); + + eprintln!("executing {cmd:?}"); + + let output = cmd.output()?; + let Some(include) = output + .stderr + .split(|x| *x == b'\n') + .skip_while(|x| x.trim_ascii() != b"#include <...> search starts here:") + .nth(1) + else { + return Err(io::Error::other( + "failed to find #include <...> search path for clang", + )); + }; + let include = Path::new(OsStr::from_bytes(include.trim_ascii())); + let include = if include.ends_with("include") + && let Some(include) = include.parent() + && include.ends_with("usr") + && let Some(include) = include.parent() + { + include + } else { + return Err(io::Error::other( + "clang #include <...> search path should end in /usr/include", + )); + }; + + Ok(PathBuf::from(include)) +} + #[expect(clippy::too_many_lines)] fn configure_cargo_cmd( nix_env: &NixEnv, + host_sysroot: &Path, target_dir: &Path, crate_dir: &Path, debug: bool, @@ -303,7 +353,6 @@ fn configure_cargo_cmd( libclang_rt, pkg_config, host_libcxx, - host_sysroot, .. } = nix_env; @@ -471,13 +520,14 @@ fn configure_cargo_cmd( fn build_wasm_codec( nix_env: &NixEnv, + host_sysroot: &Path, target_dir: &Path, crate_dir: &Path, crate_name: &str, debug: bool, verbose: bool, ) -> io::Result { - let mut cmd = configure_cargo_cmd(nix_env, target_dir, crate_dir, debug); + let mut cmd = configure_cargo_cmd(nix_env, host_sysroot, target_dir, crate_dir, debug); cmd.arg("rustc") .arg("--crate-type=cdylib") .arg("-Z") From 1592214b53662be9e8e43ca90568d0b709f39f90 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 2 Mar 2026 10:08:46 +0200 Subject: [PATCH 22/55] debug clang include path on Linux --- crates/numcodecs-wasm-builder/src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index 7fbb48295..632c691f7 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -304,6 +304,7 @@ fn find_clang_host_sysroot(nix_env: &NixEnv, flake_parent_dir: &Path) -> io::Res eprintln!("executing {cmd:?}"); let output = cmd.output()?; + eprintln!("output={:?}", String::from_utf8_lossy(&output.stderr)); let Some(include) = output .stderr .split(|x| *x == b'\n') @@ -314,7 +315,9 @@ fn find_clang_host_sysroot(nix_env: &NixEnv, flake_parent_dir: &Path) -> io::Res "failed to find #include <...> search path for clang", )); }; + eprintln!("include={:?}", String::from_utf8_lossy(include)); let include = Path::new(OsStr::from_bytes(include.trim_ascii())); + eprintln!("include={}", include.display()); let include = if include.ends_with("include") && let Some(include) = include.parent() && include.ends_with("usr") From d39194ac3e7ebdd031f47bf9fcf71158ef8fbf15 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 2 Mar 2026 10:22:13 +0200 Subject: [PATCH 23/55] more flexible sysroot finding --- crates/numcodecs-wasm-builder/src/main.rs | 40 +++++++++++++++-------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index 632c691f7..ffb4dd34f 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -304,30 +304,42 @@ fn find_clang_host_sysroot(nix_env: &NixEnv, flake_parent_dir: &Path) -> io::Res eprintln!("executing {cmd:?}"); let output = cmd.output()?; - eprintln!("output={:?}", String::from_utf8_lossy(&output.stderr)); - let Some(include) = output + let Some(full_include) = output .stderr .split(|x| *x == b'\n') .skip_while(|x| x.trim_ascii() != b"#include <...> search starts here:") .nth(1) else { - return Err(io::Error::other( - "failed to find #include <...> search path for clang", - )); + return Err(io::Error::other(format!( + "failed to find #include <...> search path for clang in {:?}", + String::from_utf8_lossy(&output.stderr) + ))); }; - eprintln!("include={:?}", String::from_utf8_lossy(include)); - let include = Path::new(OsStr::from_bytes(include.trim_ascii())); - eprintln!("include={}", include.display()); - let include = if include.ends_with("include") - && let Some(include) = include.parent() - && include.ends_with("usr") + let full_include = Path::new(OsStr::from_bytes(full_include.trim_ascii())); + let mut include = if full_include.ends_with("include") + && let Some(include) = full_include.parent() + { + include + } else { + return Err(io::Error::other(format!( + "clang #include <...> search path {} should end in /usr/.../include", + full_include.display() + ))); + }; + while !include.ends_with("usr") + && let Some(include_parent) = include.parent() + { + include = include_parent; + } + let include = if include.ends_with("usr") && let Some(include) = include.parent() { include } else { - return Err(io::Error::other( - "clang #include <...> search path should end in /usr/include", - )); + return Err(io::Error::other(format!( + "clang #include <...> search path {} should end in /usr/.../include", + full_include.display() + ))); }; Ok(PathBuf::from(include)) From 2d4d92d276551b0f334cd4a47b8f072bbb333e6e Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 2 Mar 2026 10:57:55 +0200 Subject: [PATCH 24/55] why not? --- crates/numcodecs-wasm-builder/src/main.rs | 79 ++--------------------- 1 file changed, 4 insertions(+), 75 deletions(-) diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index ffb4dd34f..5ce8346f6 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -3,12 +3,9 @@ use std::{ collections::HashMap, - env, - ffi::OsStr, - fs, io, - os::unix::ffi::OsStrExt, + env, fs, io, path::{Path, PathBuf}, - process::{Command, Stdio}, + process::Command, str::FromStr, }; @@ -72,11 +69,9 @@ fn main() -> io::Result<()> { copy_buildenv_to_crate(&crate_dir)?; let nix_env = NixEnv::new(&crate_dir)?; - let host_sysroot = find_clang_host_sysroot(&nix_env, &crate_dir)?; let wasm = build_wasm_codec( &nix_env, - &host_sysroot, &target_dir, &crate_dir, &format!("{}-wasm", args.crate_), @@ -281,74 +276,9 @@ impl NixEnv { } } -fn find_clang_host_sysroot(nix_env: &NixEnv, flake_parent_dir: &Path) -> io::Result { - let NixEnv { clang, .. } = nix_env; - - let mut cmd = Command::new("nix"); - cmd.current_dir(flake_parent_dir); - cmd.arg("develop"); - // cmd.arg("--store"); - // cmd.arg(nix_store_path); - cmd.arg("--no-update-lock-file"); - cmd.arg("--ignore-environment"); - cmd.arg("path:."); - cmd.arg("--command"); - cmd.arg(clang.join("clang")); - cmd.arg("-v"); - cmd.arg("-x"); - cmd.arg("c"); - cmd.arg("-c"); - cmd.arg("-"); - cmd.stdin(Stdio::null()); - - eprintln!("executing {cmd:?}"); - - let output = cmd.output()?; - let Some(full_include) = output - .stderr - .split(|x| *x == b'\n') - .skip_while(|x| x.trim_ascii() != b"#include <...> search starts here:") - .nth(1) - else { - return Err(io::Error::other(format!( - "failed to find #include <...> search path for clang in {:?}", - String::from_utf8_lossy(&output.stderr) - ))); - }; - let full_include = Path::new(OsStr::from_bytes(full_include.trim_ascii())); - let mut include = if full_include.ends_with("include") - && let Some(include) = full_include.parent() - { - include - } else { - return Err(io::Error::other(format!( - "clang #include <...> search path {} should end in /usr/.../include", - full_include.display() - ))); - }; - while !include.ends_with("usr") - && let Some(include_parent) = include.parent() - { - include = include_parent; - } - let include = if include.ends_with("usr") - && let Some(include) = include.parent() - { - include - } else { - return Err(io::Error::other(format!( - "clang #include <...> search path {} should end in /usr/.../include", - full_include.display() - ))); - }; - - Ok(PathBuf::from(include)) -} - #[expect(clippy::too_many_lines)] fn configure_cargo_cmd( nix_env: &NixEnv, - host_sysroot: &Path, target_dir: &Path, crate_dir: &Path, debug: bool, @@ -455,7 +385,7 @@ fn configure_cargo_cmd( cmd.arg(format!( "CXXFLAGSHOST=-isysroot {host_sysroot} -isystem {host_libcxx_include} \ -isystem {clang_include}", - host_sysroot = host_sysroot.display(), + host_sysroot = wasi_sysroot.display(), // I mean, what could go wrong? host_libcxx_include = host_libcxx.join("include").join("c++").join("v1").display(), clang_include = libclang .join("clang") @@ -535,14 +465,13 @@ fn configure_cargo_cmd( fn build_wasm_codec( nix_env: &NixEnv, - host_sysroot: &Path, target_dir: &Path, crate_dir: &Path, crate_name: &str, debug: bool, verbose: bool, ) -> io::Result { - let mut cmd = configure_cargo_cmd(nix_env, host_sysroot, target_dir, crate_dir, debug); + let mut cmd = configure_cargo_cmd(nix_env, target_dir, crate_dir, debug); cmd.arg("rustc") .arg("--crate-type=cdylib") .arg("-Z") From 2cdc5293bd412408369a4cf5f3def209751cf9b5 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 2 Mar 2026 11:10:43 +0200 Subject: [PATCH 25/55] try again --- crates/numcodecs-wasm-builder/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index 5ce8346f6..d2a5851ad 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -385,7 +385,7 @@ fn configure_cargo_cmd( cmd.arg(format!( "CXXFLAGSHOST=-isysroot {host_sysroot} -isystem {host_libcxx_include} \ -isystem {clang_include}", - host_sysroot = wasi_sysroot.display(), // I mean, what could go wrong? + host_sysroot = wasi_sysroot.join("include").join("wasm32-wasi").display(), // I mean, what could go wrong? host_libcxx_include = host_libcxx.join("include").join("c++").join("v1").display(), clang_include = libclang .join("clang") From 191de869059f27b8e9298210c56b792f612c8b31 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 2 Mar 2026 16:30:45 +0200 Subject: [PATCH 26/55] separate error for en/de-code to array without data --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 666a5f759..c5b0d52fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "6329913", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "7c38eb3", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 40b008b06..296d3d7d2 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -244,18 +244,43 @@ impl Codec for PressioCodec { Ok(data) => libpressio::PressioData::new(data), Err(data) => libpressio::PressioData::new_copied(data.view()), }; + eprintln!( + "data: {} {} {} {:?}", + data.has_data(), + data.len(), + data.ndim(), + data.dtype() + ); let compressed_data = libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); + eprintln!( + "compressed: {} {} {} {:?}", + compressed_data.has_data(), + compressed_data.len(), + compressed_data.ndim(), + compressed_data.dtype() + ); let compressed_data = compressor.compress(&data, compressed_data).map_err(|err| { PressioCodecError::PressioEncodeFailed { source: PressioCodingError(err), } })?; + eprintln!( + "compressed: {} {} {} {:?}", + compressed_data.has_data(), + compressed_data.len(), + compressed_data.ndim(), + compressed_data.dtype() + ); let Some(compressed_data) = compressed_data.clone_into_array() else { - return Err(PressioCodecError::EncodeToUnknownDtype); + if compressed_data.has_data() { + return Err(PressioCodecError::EncodeToUnknownDtype); + } + + return Err(PressioCodecError::EncodeToArrayWithoutData); }; match compressed_data { @@ -315,7 +340,11 @@ impl Codec for PressioCodec { })?; let Some(decompressed_data) = decompressed_data.clone_into_array() else { - return Err(PressioCodecError::DecodeToUnknownDtype); + if decompressed_data.has_data() { + return Err(PressioCodecError::DecodeToUnknownDtype); + } + + return Err(PressioCodecError::DecodeToArrayWithoutData); }; match decompressed_data { @@ -398,6 +427,9 @@ pub enum PressioCodecError { /// [`PressioCodec`] encoded to an unknown unsupported dtype #[error("Pressio encoded to an unknown unsupported dtype")] EncodeToUnknownDtype, + /// [`PressioCodec`] encoded to an array without data + #[error("Pressio encoded to an array without data")] + EncodeToArrayWithoutData, /// [`PressioCodec`] encoded to a bool array, which is unsupported #[error("Pressio encoded to a bool array, which is unsupported")] EncodeToBoolArray, @@ -410,6 +442,9 @@ pub enum PressioCodecError { /// [`PressioCodec`] decoded to an unknown unsupported dtype #[error("Pressio decoded to an unknown unsupported dtype")] DecodeToUnknownDtype, + /// [`PressioCodec`] decoded to an array without data + #[error("Pressio decoded to an array without data")] + DecodeToArrayWithoutData, /// [`PressioCodec`] decoded to a bool array, which is unsupported #[error("Pressio decoded to a bool array, which is unsupported")] DecodeToBoolArray, From 478c90d116cae4a78b1fef7049f908f035c011a4 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 9 Mar 2026 10:37:44 +0200 Subject: [PATCH 27/55] Enable more plugins --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 58 ++++++++++++++++----------------------- 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c5b0d52fb..490d32d41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "7c38eb3", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5e533d8", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 296d3d7d2..df38fad0f 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -240,33 +240,17 @@ impl Codec for PressioCodec { compressor: &mut libpressio::PressioCompressor, data: CowArray, ) -> Result { - let data = match data.try_into_owned_nocopy() { - Ok(data) => libpressio::PressioData::new(data), - Err(data) => libpressio::PressioData::new_copied(data.view()), - }; - eprintln!( - "data: {} {} {} {:?}", - data.has_data(), - data.len(), - data.ndim(), - data.dtype() - ); - - let compressed_data = - libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); - eprintln!( - "compressed: {} {} {} {:?}", - compressed_data.has_data(), - compressed_data.len(), - compressed_data.ndim(), - compressed_data.dtype() - ); + let compressed_data = libpressio::PressioData::new_with_shared(data, |data| { + let compressed_data = + libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); - let compressed_data = compressor.compress(&data, compressed_data).map_err(|err| { - PressioCodecError::PressioEncodeFailed { - source: PressioCodingError(err), - } + compressor.compress(data, compressed_data).map_err(|err| { + PressioCodecError::PressioEncodeFailed { + source: PressioCodingError(err), + } + }) })?; + eprintln!( "compressed: {} {} {} {:?}", compressed_data.has_data(), @@ -324,20 +308,24 @@ impl Codec for PressioCodec { compressor: &mut libpressio::PressioCompressor, encoded: CowArray, ) -> Result { - let encoded = match encoded.try_into_owned_nocopy() { - Ok(encoded) => libpressio::PressioData::new(encoded), - Err(encoded) => libpressio::PressioData::new_copied(encoded.view()), - }; + let decompressed_data = libpressio::PressioData::new_with_shared(encoded, |encoded| { + let decompressed_data = + libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); - let decompressed_data = - libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); - - let decompressed_data = compressor - .compress(&encoded, decompressed_data) + .compress(encoded, decompressed_data) .map_err(|err| PressioCodecError::PressioDecodeFailed { source: PressioCodingError(err), - })?; + }) + })?; + + eprintln!( + "decompressed: {} {} {} {:?}", + decompressed_data.has_data(), + decompressed_data.len(), + decompressed_data.ndim(), + decompressed_data.dtype() + ); let Some(decompressed_data) = decompressed_data.clone_into_array() else { if decompressed_data.has_data() { From c3bf4ed8530f1b877432d6c169351964b67939ed Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Tue, 10 Mar 2026 09:31:27 +0200 Subject: [PATCH 28/55] port pressio_register_generator to CMake --- Cargo.toml | 2 +- crates/numcodecs-wasm-builder/buildenv/flake.nix | 2 -- crates/numcodecs-wasm-builder/src/main.rs | 14 -------------- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 490d32d41..0f9fb2837 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -146,7 +146,7 @@ zstd = { version = "0.13", default-features = false } zstd-sys = { version = "2.0.16", default-features = false } # git third-party dependencies with non-upstream fixes -wasm_component_layer = { git = "https://github.com/juntyr/wasm_component_layer.git", rev = "e923536", version = "0.1", default-features = false } +wasm_component_layer = { git = "https://github.com/juntyr/wasm_component_layer.git", rev = "1b8708e", version = "0.1", default-features = false } [workspace.lints.rust] unsafe_code = "deny" diff --git a/crates/numcodecs-wasm-builder/buildenv/flake.nix b/crates/numcodecs-wasm-builder/buildenv/flake.nix index 22ce9af96..b02efb029 100644 --- a/crates/numcodecs-wasm-builder/buildenv/flake.nix +++ b/crates/numcodecs-wasm-builder/buildenv/flake.nix @@ -61,7 +61,6 @@ packages = [ (pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain) pkgs."llvmPackages_${llvmVersion}".libclang - pkgs."llvmPackages_${llvmVersion}".libcxx wasi-sysroot libclang_rt pkgs.cmake @@ -85,7 +84,6 @@ MY_WASM_OPT = "${pkgs.binaryen}/bin/wasm-opt"; MY_PKG_CONFIG = "${pkgs.pkg-config}/bin/pkg-config"; MY_PYTHON3 = "${pkgs.python3}/bin/python3"; - MY_HOST_LIBCXX = "${pkgs."llvmPackages_${llvmVersion}".libcxx.dev}"; }; }; }); diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index d2a5851ad..329d6acf0 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -203,7 +203,6 @@ struct NixEnv { pkg_config: PathBuf, #[expect(dead_code)] python3: PathBuf, - host_libcxx: PathBuf, } impl NixEnv { @@ -271,7 +270,6 @@ impl NixEnv { wasm_opt: try_read_env(&env, "MY_WASM_OPT")?, pkg_config: try_read_env(&env, "MY_PKG_CONFIG")?, python3: try_read_env(&env, "MY_PYTHON3")?, - host_libcxx: try_read_env(&env, "MY_HOST_LIBCXX")?, }) } } @@ -297,7 +295,6 @@ fn configure_cargo_cmd( wasi_sysroot, libclang_rt, pkg_config, - host_libcxx, .. } = nix_env; @@ -382,17 +379,6 @@ fn configure_cargo_cmd( cpp_include_path = crate_dir.join("include.hpp").display(), debug = if debug { "-g" } else { "" }, )); - cmd.arg(format!( - "CXXFLAGSHOST=-isysroot {host_sysroot} -isystem {host_libcxx_include} \ - -isystem {clang_include}", - host_sysroot = wasi_sysroot.join("include").join("wasm32-wasi").display(), // I mean, what could go wrong? - host_libcxx_include = host_libcxx.join("include").join("c++").join("v1").display(), - clang_include = libclang - .join("clang") - .join(llvm_version) - .join("include") - .display(), - )); cmd.arg(format!( "BINDGEN_EXTRA_CLANG_ARGS=--target=wasm32-wasip1 -nodefaultlibs -resource-dir \ {resource_dir} --sysroot={wasi_sysroot} -isystem {wasm32_wasi_cxx_include} -isystem \ From ecded8056ef0edbc47f05babe7b5c10121c3f9ac Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Tue, 10 Mar 2026 09:33:15 +0200 Subject: [PATCH 29/55] Fix git rev --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0f9fb2837..1f38002b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5e533d8", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "1b8708e", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy @@ -146,7 +146,7 @@ zstd = { version = "0.13", default-features = false } zstd-sys = { version = "2.0.16", default-features = false } # git third-party dependencies with non-upstream fixes -wasm_component_layer = { git = "https://github.com/juntyr/wasm_component_layer.git", rev = "1b8708e", version = "0.1", default-features = false } +wasm_component_layer = { git = "https://github.com/juntyr/wasm_component_layer.git", rev = "e923536", version = "0.1", default-features = false } [workspace.lints.rust] unsafe_code = "deny" From d235c6b6e0f2c78b8200de7ac302d5af8cd8e652 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Tue, 10 Mar 2026 09:50:47 +0200 Subject: [PATCH 30/55] add LIBPRESSIO_WITH_EXTERNAL option --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 1f38002b8..acee0f2d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "1b8708e", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "6b65157", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From 72b9c7785d12761e1ae46d82f19ed2b7d31325e0 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 11 Mar 2026 10:53:12 +0200 Subject: [PATCH 31/55] Start bringing codec config closer to existing libpressio numcodecs interface --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 412 ++++++++++++++++++++++++++++---------- 2 files changed, 304 insertions(+), 110 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index acee0f2d2..310f565f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "6b65157", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "ec5df41", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index df38fad0f..fde1210f6 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -17,23 +17,17 @@ //! //! libpressio codec wrapper for the [`numcodecs`] API. -use std::{ - borrow::Cow, - collections::BTreeMap, - sync::{LazyLock, Mutex}, -}; +use std::{borrow::Cow, collections::BTreeMap, sync::Mutex}; -use ndarray::{CowArray, IxDyn}; +use ndarray::{ArrayView, ArrayViewMut, CowArray, IxDyn}; use numcodecs::{ AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, StaticCodecConfig, StaticCodecVersion, }; -use schemars::{JsonSchema, Schema, SchemaGenerator}; +use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; -static PRESSIO: LazyLock = LazyLock::new(Pressio::new); - #[derive(Clone, Serialize, Deserialize, JsonSchema)] #[schemars(deny_unknown_fields)] /// Pressio codec which applies the identity function, i.e. passes through the @@ -56,12 +50,10 @@ pub struct PressioCompressor { impl Clone for PressioCompressor { #[expect(clippy::unwrap_used)] fn clone(&self) -> Self { - let mut compressor = { - let mut pressio = PRESSIO.get_or_unwrap().lock().unwrap(); - pressio - .get_compressor(self.format.compressor.as_str()) - .unwrap() - }; + let mut pressio = libpressio::Pressio::new().unwrap(); + let mut compressor = pressio + .get_compressor(self.format.compressor_id.as_str()) + .unwrap(); let options = self.compressor.lock().unwrap().get_options().unwrap(); compressor.set_options(&options).unwrap(); @@ -80,89 +72,102 @@ impl Serialize for PressioCompressor { impl<'de> Deserialize<'de> for PressioCompressor { fn deserialize>(deserializer: D) -> Result { - // TODO: better error handling - let format = PressioCompressorFormat::deserialize(deserializer)?; - let compressor = { - let pressio = PRESSIO - .get() - .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; - let mut pressio = pressio.lock().map_err(serde::de::Error::custom)?; - pressio - .get_compressor(format.compressor.as_str()) - .map_err(|err| { - let supported_compressors = pressio.supported_compressors().map_or_else( - |_| String::from(""), - |x| { - x.iter() - .map(|x| format!("`{x}`")) - .collect::>() - .join(", ") - }, - ); - - serde::de::Error::custom(format_args!( - "{}, choose one of: {}", - err.message, supported_compressors - )) - })? - }; - let mut options = compressor - .get_options() - .map_err(|err| serde::de::Error::custom(err.message))?; - - for (key, value) in &format.options { - options = options - .set( - key, - match value { - PressioOption::Bool(x) => libpressio::PressioOption::bool(Some(*x)), - PressioOption::U8(x) => libpressio::PressioOption::uint8(Some(*x)), - PressioOption::I8(x) => libpressio::PressioOption::int8(Some(*x)), - PressioOption::U16(x) => libpressio::PressioOption::uint16(Some(*x)), - PressioOption::I16(x) => libpressio::PressioOption::int16(Some(*x)), - PressioOption::U32(x) => libpressio::PressioOption::uint32(Some(*x)), - PressioOption::I32(x) => libpressio::PressioOption::int32(Some(*x)), - PressioOption::U64(x) => libpressio::PressioOption::uint64(Some(*x)), - PressioOption::I64(x) => libpressio::PressioOption::int64(Some(*x)), - PressioOption::F32(x) => libpressio::PressioOption::float32(Some(*x)), - PressioOption::F64(x) => libpressio::PressioOption::float64(Some(*x)), + fn convert_to_pressio_options( + config: &BTreeMap, + ) -> Result { + let mut options = libpressio::PressioOptions::new()?; + + let mut entries = vec![(vec![], config)]; + + while let Some((path, entry)) = entries.pop() { + for (key, value) in entry { + let option = match value { + PressioOption::None(None) => Option::None, + PressioOption::Bool(x) => Some(libpressio::PressioOption::bool(Some(*x))), + PressioOption::U8(x) => Some(libpressio::PressioOption::uint8(Some(*x))), + PressioOption::I8(x) => Some(libpressio::PressioOption::int8(Some(*x))), + PressioOption::U16(x) => Some(libpressio::PressioOption::uint16(Some(*x))), + PressioOption::I16(x) => Some(libpressio::PressioOption::int16(Some(*x))), + PressioOption::U32(x) => Some(libpressio::PressioOption::uint32(Some(*x))), + PressioOption::I32(x) => Some(libpressio::PressioOption::int32(Some(*x))), + PressioOption::U64(x) => Some(libpressio::PressioOption::uint64(Some(*x))), + PressioOption::I64(x) => Some(libpressio::PressioOption::int64(Some(*x))), + PressioOption::F32(x) => Some(libpressio::PressioOption::float32(Some(*x))), + PressioOption::F64(x) => Some(libpressio::PressioOption::float64(Some(*x))), PressioOption::String(x) => { - libpressio::PressioOption::string(Some(x.clone())) + Some(libpressio::PressioOption::string(Some(x.clone()))) } PressioOption::VecString(x) => { - libpressio::PressioOption::vec_string(Some(x.clone())) + Some(libpressio::PressioOption::vec_string(Some(x.clone()))) } + PressioOption::Nested(entry) => { + let mut nested_path = path.clone(); + nested_path.push(key.clone()); + entries.push((nested_path, entry)); + continue; + } + }; + + let name = if path.is_empty() { + key.clone() + } else { + format!("{path}:{key}", path = path.join("/")) + }; + + if let Some(option) = option { + options = options.set(name, option)?; + } + } + } + + Ok(options) + } + + // TODO: better error handling + let format = PressioCompressorFormat::deserialize(deserializer)?; + + let mut pressio = libpressio::Pressio::new() + .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + let mut compressor = pressio + .get_compressor(format.compressor_id.as_str()) + .map_err(|err| { + let supported_compressors = pressio.supported_compressors().map_or_else( + |_| String::from(""), + |x| { + x.iter() + .map(|x| format!("`{x}`")) + .collect::>() + .join(", ") }, - ) - .map_err(|err| serde::de::Error::custom(err.message))?; + ); + + serde::de::Error::custom(format_args!( + "{}, choose one of: {}", + err.message, supported_compressors + )) + })?; + + if let Some(name) = &format.name { + compressor + .set_name(name) + .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; } - let mut format = format; - if let Ok(format_options) = options.get_options() { - format.options = format_options - .into_iter() - .filter_map(|(k, v)| match v { - libpressio::PressioOption::bool(Some(x)) => Some((k, PressioOption::Bool(x))), - libpressio::PressioOption::int8(Some(x)) => Some((k, PressioOption::I8(x))), - libpressio::PressioOption::int16(Some(x)) => Some((k, PressioOption::I16(x))), - libpressio::PressioOption::int32(Some(x)) => Some((k, PressioOption::I32(x))), - libpressio::PressioOption::int64(Some(x)) => Some((k, PressioOption::I64(x))), - libpressio::PressioOption::uint8(Some(x)) => Some((k, PressioOption::U8(x))), - libpressio::PressioOption::uint16(Some(x)) => Some((k, PressioOption::U16(x))), - libpressio::PressioOption::uint32(Some(x)) => Some((k, PressioOption::U32(x))), - libpressio::PressioOption::uint64(Some(x)) => Some((k, PressioOption::U64(x))), - libpressio::PressioOption::float32(Some(x)) => Some((k, PressioOption::F32(x))), - libpressio::PressioOption::float64(Some(x)) => Some((k, PressioOption::F64(x))), - libpressio::PressioOption::string(Some(x)) => { - Some((k, PressioOption::String(x))) - } - // FIXME: seems to return strings as a single joined string - libpressio::PressioOption::vec_string(Some(x)) => { - Some((k, PressioOption::VecString(x))) - } - _ => None, - }) - .collect(); + let early_options = convert_to_pressio_options(&format.early_config) + .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + compressor + .set_options(&early_options) + .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + + let _options_template = compressor + .get_options() + .map_err(|err| serde::de::Error::custom(err.message))?; + + if !format.compressor_config.is_empty() { + // TODO + return Err(serde::de::Error::custom( + "compressor_config is not yet supported", + )); } Ok(Self { @@ -185,10 +190,17 @@ impl JsonSchema for PressioCompressor { #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(rename = "PressioCompressor")] struct PressioCompressorFormat { - compressor: String, - // TODO: flatten + /// The id of the compressor + compressor_id: String, + /// Configuration for the structure of the compressor #[serde(default)] - options: BTreeMap, + early_config: BTreeMap, + /// Configuration for the compressor + #[serde(default)] + compressor_config: BTreeMap, + /// Optional name for the compressor when used in hierarchical mode + #[serde(default)] + name: Option, } #[expect(missing_docs)] @@ -196,6 +208,7 @@ struct PressioCompressorFormat { #[serde(untagged)] /// Pressio option value pub enum PressioOption { + None(None), Bool(bool), U8(u8), I8(i8), @@ -209,26 +222,50 @@ pub enum PressioOption { F64(f64), String(String), VecString(Vec), + Nested(BTreeMap), } -struct Pressio { - pressio: Result, libpressio::PressioError>, +#[derive(Copy, Clone, Debug)] +/// Equivalent of `Option::None` +pub struct None; + +impl Serialize for None { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_none() + } } -impl Pressio { - fn new() -> Self { - Self { - pressio: libpressio::Pressio::new().map(Mutex::new), +impl<'de> Deserialize<'de> for None { + fn deserialize>(deserializer: D) -> Result { + enum Never {} + + impl<'de> Deserialize<'de> for Never { + fn deserialize>(_deserializer: D) -> Result { + Err(serde::de::Error::custom("never")) + } } + + match Option::::deserialize(deserializer) { + Ok(Option::Some(x)) => match x {}, + Ok(Option::None) => Ok(Self), + Err(err) => Err(err), + } + } +} + +impl JsonSchema for None { + fn schema_name() -> Cow<'static, str> { + Cow::Borrowed("null") } - const fn get(&self) -> Result<&Mutex, &libpressio::PressioError> { - self.pressio.as_ref() + fn inline_schema() -> bool { + true } - #[expect(clippy::unwrap_used)] - fn get_or_unwrap(&self) -> &Mutex { - self.pressio.as_ref().unwrap() + fn json_schema(_generator: &mut SchemaGenerator) -> Schema { + json_schema!({ + "type": "null" + }) } } @@ -371,15 +408,172 @@ impl Codec for PressioCodec { } } + #[expect(clippy::too_many_lines)] // FIXME fn decode_into( &self, encoded: AnyArrayView, - mut decoded: AnyArrayViewMut, + decoded: AnyArrayViewMut, ) -> Result<(), Self::Error> { - // TODO: optimize - let decoded_in = self.decode(encoded.cow())?; + fn decompress_typed( + compressor: &mut libpressio::PressioCompressor, + encoded: ArrayView, + decoded_dtype: libpressio::PressioDtype, + decoded_shape: &[usize], + ) -> Result { + libpressio::PressioData::new_with_shared(encoded, |encoded| { + let decompressed_data = + libpressio::PressioData::new_empty(decoded_dtype, decoded_shape); + + compressor + .compress(encoded, decompressed_data) + .map_err(|err| PressioCodecError::PressioDecodeFailed { + source: PressioCodingError(err), + }) + }) + } + + fn decode_into_typed( + decompressed_data: &libpressio::PressioData, + mut decoded: ArrayViewMut, + ) -> Result<(), PressioCodecError> { + eprintln!( + "decompressed into: {} {} {} {:?}", + decompressed_data.has_data(), + decompressed_data.len(), + decompressed_data.ndim(), + decompressed_data.dtype() + ); + + if !decompressed_data.has_data() { + return Err(PressioCodecError::DecodeToArrayWithoutData); + } + + let dtype = match ::DTYPE { + libpressio::PressioDtype::Bool => { + return Err(PressioCodecError::DecodeToBoolArray); + } + libpressio::PressioDtype::Byte | libpressio::PressioDtype::U8 => AnyArrayDType::U8, + libpressio::PressioDtype::U16 => AnyArrayDType::U16, + libpressio::PressioDtype::U32 => AnyArrayDType::U32, + libpressio::PressioDtype::U64 => AnyArrayDType::U64, + libpressio::PressioDtype::I8 => AnyArrayDType::I8, + libpressio::PressioDtype::I16 => AnyArrayDType::I16, + libpressio::PressioDtype::I32 => AnyArrayDType::I32, + libpressio::PressioDtype::I64 => AnyArrayDType::I64, + libpressio::PressioDtype::F32 => AnyArrayDType::F32, + libpressio::PressioDtype::F64 => AnyArrayDType::F64, + }; + let decompressed_dtype = match decompressed_data.dtype() { + Option::None => return Err(PressioCodecError::DecodeToUnknownDtype), + Some(libpressio::PressioDtype::Bool) => { + return Err(PressioCodecError::DecodeToBoolArray); + } + Some(libpressio::PressioDtype::Byte | libpressio::PressioDtype::U8) => { + AnyArrayDType::U8 + } + Some(libpressio::PressioDtype::U16) => AnyArrayDType::U16, + Some(libpressio::PressioDtype::U32) => AnyArrayDType::U32, + Some(libpressio::PressioDtype::U64) => AnyArrayDType::U64, + Some(libpressio::PressioDtype::I8) => AnyArrayDType::I8, + Some(libpressio::PressioDtype::I16) => AnyArrayDType::I16, + Some(libpressio::PressioDtype::I32) => AnyArrayDType::I32, + Some(libpressio::PressioDtype::I64) => AnyArrayDType::I64, + Some(libpressio::PressioDtype::F32) => AnyArrayDType::F32, + Some(libpressio::PressioDtype::F64) => AnyArrayDType::F64, + }; + + if dtype != decompressed_dtype { + return Err(PressioCodecError::MismatchedDecodeIntoArray { + source: AnyArrayAssignError::DTypeMismatch { + src: decompressed_dtype, + dst: dtype, + }, + }); + } - Ok(decoded.assign(&decoded_in)?) + if decompressed_data + .with_shared::(decoded.dim(), |decompressed| { + decoded.assign(&decompressed); + }) + .is_none() + { + return Err(PressioCodecError::MismatchedDecodeIntoArray { + source: AnyArrayAssignError::ShapeMismatch { + src: decompressed_data.shape(), + dst: decoded.shape().to_vec(), + }, + }); + } + + Ok(()) + } + + let Ok(mut compressor) = self.compressor.compressor.lock() else { + return Err(PressioCodecError::PressioPoisonedMutex); + }; + + let decoded_dtype = match decoded.dtype() { + AnyArrayDType::U8 => libpressio::PressioDtype::U8, + AnyArrayDType::U16 => libpressio::PressioDtype::U16, + AnyArrayDType::U32 => libpressio::PressioDtype::U32, + AnyArrayDType::U64 => libpressio::PressioDtype::U64, + AnyArrayDType::I8 => libpressio::PressioDtype::I8, + AnyArrayDType::I16 => libpressio::PressioDtype::I16, + AnyArrayDType::I32 => libpressio::PressioDtype::I32, + AnyArrayDType::I64 => libpressio::PressioDtype::I64, + AnyArrayDType::F32 => libpressio::PressioDtype::F32, + AnyArrayDType::F64 => libpressio::PressioDtype::F64, + decoded_dtype => return Err(PressioCodecError::UnsupportedDtype(decoded_dtype)), + }; + let decoded_shape = decoded.shape(); + + let decompressed_data = match encoded { + AnyArrayView::U8(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::U16(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::U32(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::U64(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::I8(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::I16(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::I32(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::I64(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::F32(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::F64(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + encoded => return Err(PressioCodecError::UnsupportedDtype(encoded.dtype())), + }?; + + match decoded { + AnyArrayViewMut::U8(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::U16(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::U32(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::U64(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::I8(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::I16(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::I32(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::I64(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::F32(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::F64(decoded) => decode_into_typed(&decompressed_data, decoded), + decoded => Err(PressioCodecError::UnsupportedDtype(decoded.dtype())), + } } } From 4725ed569819861a66f75cd6852889825adfbcbb Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 11 Mar 2026 11:52:07 +0200 Subject: [PATCH 32/55] Fix clippy lint --- codecs/pressio/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index fde1210f6..7c2723349 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -222,7 +222,7 @@ pub enum PressioOption { F64(f64), String(String), VecString(Vec), - Nested(BTreeMap), + Nested(BTreeMap), } #[derive(Copy, Clone, Debug)] From 6f8fce87fe2d0fcb3f00332f9309302b63a3dacd Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 12 Mar 2026 12:11:19 +0200 Subject: [PATCH 33/55] Progress with serialising the compressor config --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 197 +++++++++++++++++++++++++++++++------- 2 files changed, 165 insertions(+), 34 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 310f565f5..15c8c26d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "ec5df41", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "bed5dc3", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 7c2723349..a664e0acc 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -17,7 +17,11 @@ //! //! libpressio codec wrapper for the [`numcodecs`] API. -use std::{borrow::Cow, collections::BTreeMap, sync::Mutex}; +use std::{ + borrow::Cow, + collections::{BTreeMap, btree_map::Entry}, + sync::Mutex, +}; use ndarray::{ArrayView, ArrayViewMut, CowArray, IxDyn}; use numcodecs::{ @@ -43,39 +47,137 @@ pub struct PressioCodec { /// Pressio compressor pub struct PressioCompressor { - format: PressioCompressorFormat, compressor: Mutex, + compressor_id: String, + early_config: BTreeMap, + name: Option, } impl Clone for PressioCompressor { #[expect(clippy::unwrap_used)] fn clone(&self) -> Self { let mut pressio = libpressio::Pressio::new().unwrap(); - let mut compressor = pressio - .get_compressor(self.format.compressor_id.as_str()) - .unwrap(); + let mut compressor = pressio.get_compressor(self.compressor_id.as_str()).unwrap(); + if let Some(name) = &self.name { + compressor.set_name(name).unwrap(); + } let options = self.compressor.lock().unwrap().get_options().unwrap(); compressor.set_options(&options).unwrap(); Self { - format: self.format.clone(), compressor: Mutex::new(compressor), + compressor_id: self.compressor_id.clone(), + early_config: self.early_config.clone(), + name: self.name.clone(), } } } impl Serialize for PressioCompressor { fn serialize(&self, serializer: S) -> Result { - self.format.serialize(serializer) + fn convert_from_pressio_options( + options: BTreeMap, + ) -> Result, E> { + let mut config = BTreeMap::new(); + + for (name, option) in options { + let value = match option { + libpressio::PressioOption::bool(Some(x)) => PressioOption::Bool(x), + libpressio::PressioOption::int8(Some(x)) => PressioOption::I8(x), + libpressio::PressioOption::int16(Some(x)) => PressioOption::I16(x), + libpressio::PressioOption::int32(Some(x)) => PressioOption::I32(x), + libpressio::PressioOption::int64(Some(x)) => PressioOption::I64(x), + libpressio::PressioOption::uint8(Some(x)) => PressioOption::U8(x), + libpressio::PressioOption::uint16(Some(x)) => PressioOption::U16(x), + libpressio::PressioOption::uint32(Some(x)) => PressioOption::U32(x), + libpressio::PressioOption::uint64(Some(x)) => PressioOption::U64(x), + libpressio::PressioOption::float32(Some(x)) => PressioOption::F32(x), + libpressio::PressioOption::float64(Some(x)) => PressioOption::F64(x), + libpressio::PressioOption::string(Some(x)) => PressioOption::String(x), + // FIXME: seems to return strings as a single joined string + libpressio::PressioOption::vec_string(Some(x)) => PressioOption::VecString(x), + libpressio::PressioOption::data(_) + | libpressio::PressioOption::user_ptr(_) + | libpressio::PressioOption::unset + | _ /* non-exhaustive */ => continue, + }; + + let Some(nested_name) = name.strip_prefix('/') else { + // global option + if config.insert(name.clone(), value).is_some() { + return Err(serde::ser::Error::custom(format!( + "duplicate option {name:?}" + ))); + } + continue; + }; + + // hierarchical option + let mut parts = nested_name.split(':').peekable(); + + let Some(first) = parts.next() else { + return Err(serde::ser::Error::custom(format!( + "invalid hierarchical config name {name:?}" + ))); + }; + let paths = first.split('/'); + + if parts.peek().is_none() { + return Err(serde::ser::Error::custom(format!( + "invalid hierarchical config name {name:?}" + ))); + } + let option_name = parts.map(String::from).collect::>().join(":"); + + let mut it = &mut config; + for path in paths { + if let Entry::Vacant(entry) = it.entry(String::from(path)) { + entry.insert(PressioOption::Nested(BTreeMap::new())); + } + + let Some(PressioOption::Nested(entry)) = it.get_mut(path) else { + return Err(serde::ser::Error::custom(format!( + "duplicate option {path:?}" + ))); + }; + it = entry; + } + if it.insert(option_name.clone(), value).is_some() { + return Err(serde::ser::Error::custom(format!( + "duplicate option {option_name:?}" + ))); + } + } + + Ok(config) + } + + let options = { + let compressor = self.compressor.lock().map_err(serde::ser::Error::custom)?; + compressor + .get_options() + .map_err(serde::ser::Error::custom)? + }; + let options = options.get_options().map_err(serde::ser::Error::custom)?; + + PressioCompressorBorrowedFormat { + compressor_id: self.compressor_id.as_str(), + early_config: &self.early_config, + compressor_config: &convert_from_pressio_options(options)?, + name: self.name.as_deref(), + } + .serialize(serializer) } } impl<'de> Deserialize<'de> for PressioCompressor { fn deserialize>(deserializer: D) -> Result { - fn convert_to_pressio_options( + fn convert_to_pressio_options( config: &BTreeMap, - ) -> Result { - let mut options = libpressio::PressioOptions::new()?; + template: Option<&libpressio::PressioOptions>, + ) -> Result { + let mut options = + libpressio::PressioOptions::new().map_err(serde::de::Error::custom)?; let mut entries = vec![(vec![], config)]; @@ -114,8 +216,25 @@ impl<'de> Deserialize<'de> for PressioCompressor { format!("{path}:{key}", path = path.join("/")) }; + if let Some(template) = template { + if !template + .has_option(&name) + .map_err(serde::de::Error::custom)? + { + return Err(serde::de::Error::custom(format!( + "unknown compressor configuration option: {name:?}" + ))); + } + } + + // TODO: handle conversion and type errors + // TODO: check if the options were actually set + // (e.g. compressor names are validated and fallback to noop) + if let Some(option) = option { - options = options.set(name, option)?; + options + .set(name, option) + .map_err(serde::de::Error::custom)?; } } } @@ -124,10 +243,9 @@ impl<'de> Deserialize<'de> for PressioCompressor { } // TODO: better error handling - let format = PressioCompressorFormat::deserialize(deserializer)?; + let format = PressioCompressorOwnedFormat::deserialize(deserializer)?; - let mut pressio = libpressio::Pressio::new() - .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + let mut pressio = libpressio::Pressio::new().map_err(serde::de::Error::custom)?; let mut compressor = pressio .get_compressor(format.compressor_id.as_str()) .map_err(|err| { @@ -150,46 +268,43 @@ impl<'de> Deserialize<'de> for PressioCompressor { if let Some(name) = &format.name { compressor .set_name(name) - .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + .map_err(serde::de::Error::custom)?; } - let early_options = convert_to_pressio_options(&format.early_config) - .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + let early_options = convert_to_pressio_options(&format.early_config, Option::None)?; compressor .set_options(&early_options) - .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; - - let _options_template = compressor - .get_options() - .map_err(|err| serde::de::Error::custom(err.message))?; + .map_err(serde::de::Error::custom)?; + let options_template = compressor.get_options().map_err(serde::de::Error::custom)?; - if !format.compressor_config.is_empty() { - // TODO - return Err(serde::de::Error::custom( - "compressor_config is not yet supported", - )); - } + let options = + convert_to_pressio_options(&format.compressor_config, Some(&options_template))?; + compressor + .set_options(&options) + .map_err(serde::de::Error::custom)?; Ok(Self { - format, compressor: Mutex::new(compressor), + compressor_id: format.compressor_id, + early_config: format.early_config, + name: format.name, }) } } impl JsonSchema for PressioCompressor { fn schema_name() -> Cow<'static, str> { - PressioCompressorFormat::schema_name() + PressioCompressorOwnedFormat::schema_name() } fn json_schema(generator: &mut SchemaGenerator) -> Schema { - PressioCompressorFormat::json_schema(generator) + PressioCompressorOwnedFormat::json_schema(generator) } } -#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +#[derive(Debug, Deserialize, JsonSchema)] #[serde(rename = "PressioCompressor")] -struct PressioCompressorFormat { +struct PressioCompressorOwnedFormat { /// The id of the compressor compressor_id: String, /// Configuration for the structure of the compressor @@ -203,6 +318,22 @@ struct PressioCompressorFormat { name: Option, } +#[derive(Debug, Serialize)] +#[serde(rename = "PressioCompressor")] +struct PressioCompressorBorrowedFormat<'a> { + /// The id of the compressor + compressor_id: &'a str, + /// Configuration for the structure of the compressor + #[serde(default)] + early_config: &'a BTreeMap, + /// Configuration for the compressor + #[serde(default)] + compressor_config: &'a BTreeMap, + /// Optional name for the compressor when used in hierarchical mode + #[serde(default)] + name: Option<&'a str>, +} + #[expect(missing_docs)] #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(untagged)] From 5dd34c66cf55f03bc3ec8f25640cb59846ed4a10 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Fri, 13 Mar 2026 09:51:52 +0200 Subject: [PATCH 34/55] small code updates --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 15c8c26d0..a212e2363 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "bed5dc3", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "095bd73", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index a664e0acc..6e2580128 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -76,11 +76,16 @@ impl Clone for PressioCompressor { impl Serialize for PressioCompressor { fn serialize(&self, serializer: S) -> Result { fn convert_from_pressio_options( - options: BTreeMap, + options: impl Iterator, Option)>, ) -> Result, E> { let mut config = BTreeMap::new(); for (name, option) in options { + // skip invalid option names and values + let (Some(name), Some(option)) = (name, option) else { + continue; + }; + let value = match option { libpressio::PressioOption::bool(Some(x)) => PressioOption::Bool(x), libpressio::PressioOption::int8(Some(x)) => PressioOption::I8(x), @@ -158,12 +163,11 @@ impl Serialize for PressioCompressor { .get_options() .map_err(serde::ser::Error::custom)? }; - let options = options.get_options().map_err(serde::ser::Error::custom)?; PressioCompressorBorrowedFormat { compressor_id: self.compressor_id.as_str(), early_config: &self.early_config, - compressor_config: &convert_from_pressio_options(options)?, + compressor_config: &convert_from_pressio_options(options.iter())?, name: self.name.as_deref(), } .serialize(serializer) @@ -249,7 +253,7 @@ impl<'de> Deserialize<'de> for PressioCompressor { let mut compressor = pressio .get_compressor(format.compressor_id.as_str()) .map_err(|err| { - let supported_compressors = pressio.supported_compressors().map_or_else( + let supported_compressors = libpressio::supported_compressors().map_or_else( |_| String::from(""), |x| { x.iter() From 4eb68b3d7f0293583f5647b36855ce052f6f46c3 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Fri, 13 Mar 2026 13:11:06 +0200 Subject: [PATCH 35/55] fix some config bugs --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 62 +++++++++++++++------------------------ 2 files changed, 25 insertions(+), 39 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a212e2363..d047975c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "095bd73", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "d928ca7", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 6e2580128..1bb50d640 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -101,6 +101,8 @@ impl Serialize for PressioCompressor { libpressio::PressioOption::string(Some(x)) => PressioOption::String(x), // FIXME: seems to return strings as a single joined string libpressio::PressioOption::vec_string(Some(x)) => PressioOption::VecString(x), + libpressio::PressioOption::dtype(Some(x)) => PressioOption::String(format!("{x}")), + libpressio::PressioOption::thread_safety(Some(x)) => PressioOption::String(format!("{x}")), libpressio::PressioOption::data(_) | libpressio::PressioOption::user_ptr(_) | libpressio::PressioOption::unset @@ -111,7 +113,7 @@ impl Serialize for PressioCompressor { // global option if config.insert(name.clone(), value).is_some() { return Err(serde::ser::Error::custom(format!( - "duplicate option {name:?}" + "duplicate global option: {name:?}" ))); } continue; @@ -142,14 +144,14 @@ impl Serialize for PressioCompressor { let Some(PressioOption::Nested(entry)) = it.get_mut(path) else { return Err(serde::ser::Error::custom(format!( - "duplicate option {path:?}" + "duplicate option nesting: {path:?} in {name:?}" ))); }; it = entry; } if it.insert(option_name.clone(), value).is_some() { return Err(serde::ser::Error::custom(format!( - "duplicate option {option_name:?}" + "duplicate nested option: {option_name:?} in {name:?}" ))); } } @@ -175,6 +177,7 @@ impl Serialize for PressioCompressor { } impl<'de> Deserialize<'de> for PressioCompressor { + #[expect(clippy::too_many_lines)] // FIXME fn deserialize>(deserializer: D) -> Result { fn convert_to_pressio_options( config: &BTreeMap, @@ -217,25 +220,32 @@ impl<'de> Deserialize<'de> for PressioCompressor { let name = if path.is_empty() { key.clone() } else { - format!("{path}:{key}", path = path.join("/")) + format!("/{path}:{key}", path = path.join("/")) }; if let Some(template) = template { - if !template - .has_option(&name) - .map_err(serde::de::Error::custom)? - { + let Some(option_template) = + template.get(&name).map_err(serde::de::Error::custom)? + else { return Err(serde::de::Error::custom(format!( "unknown compressor configuration option: {name:?}" ))); - } - } + }; - // TODO: handle conversion and type errors - // TODO: check if the options were actually set - // (e.g. compressor names are validated and fallback to noop) + options + .set(&name, option_template.copy_type_only()) + .map_err(serde::de::Error::custom)?; - if let Some(option) = option { + if let Some(option) = option { + options + .set_with_cast( + name, + option, + libpressio::PressioConversionSafety::Special, + ) + .map_err(serde::de::Error::custom)?; + } + } else if let Some(option) = option { options .set(name, option) .map_err(serde::de::Error::custom)?; @@ -423,14 +433,6 @@ impl Codec for PressioCodec { }) })?; - eprintln!( - "compressed: {} {} {} {:?}", - compressed_data.has_data(), - compressed_data.len(), - compressed_data.ndim(), - compressed_data.dtype() - ); - let Some(compressed_data) = compressed_data.clone_into_array() else { if compressed_data.has_data() { return Err(PressioCodecError::EncodeToUnknownDtype); @@ -491,14 +493,6 @@ impl Codec for PressioCodec { }) })?; - eprintln!( - "decompressed: {} {} {} {:?}", - decompressed_data.has_data(), - decompressed_data.len(), - decompressed_data.ndim(), - decompressed_data.dtype() - ); - let Some(decompressed_data) = decompressed_data.clone_into_array() else { if decompressed_data.has_data() { return Err(PressioCodecError::DecodeToUnknownDtype); @@ -571,14 +565,6 @@ impl Codec for PressioCodec { decompressed_data: &libpressio::PressioData, mut decoded: ArrayViewMut, ) -> Result<(), PressioCodecError> { - eprintln!( - "decompressed into: {} {} {} {:?}", - decompressed_data.has_data(), - decompressed_data.len(), - decompressed_data.ndim(), - decompressed_data.dtype() - ); - if !decompressed_data.has_data() { return Err(PressioCodecError::DecodeToArrayWithoutData); } From 9a77036836fd69405fc3d8c4495d57db37a9ec91 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sat, 14 Mar 2026 08:51:26 +0200 Subject: [PATCH 36/55] include metric results in config and add help when deserialising options fails --- codecs/pressio/src/lib.rs | 95 ++++++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 27 deletions(-) diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 1bb50d640..8fcd29d35 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -113,7 +113,7 @@ impl Serialize for PressioCompressor { // global option if config.insert(name.clone(), value).is_some() { return Err(serde::ser::Error::custom(format!( - "duplicate global option: {name:?}" + "duplicate global option: `{name}`" ))); } continue; @@ -124,14 +124,14 @@ impl Serialize for PressioCompressor { let Some(first) = parts.next() else { return Err(serde::ser::Error::custom(format!( - "invalid hierarchical config name {name:?}" + "invalid hierarchical config name `{name}`" ))); }; let paths = first.split('/'); if parts.peek().is_none() { return Err(serde::ser::Error::custom(format!( - "invalid hierarchical config name {name:?}" + "invalid hierarchical config name `{name}`" ))); } let option_name = parts.map(String::from).collect::>().join(":"); @@ -144,14 +144,14 @@ impl Serialize for PressioCompressor { let Some(PressioOption::Nested(entry)) = it.get_mut(path) else { return Err(serde::ser::Error::custom(format!( - "duplicate option nesting: {path:?} in {name:?}" + "duplicate option nesting: `{path}` in `{name}`" ))); }; it = entry; } if it.insert(option_name.clone(), value).is_some() { return Err(serde::ser::Error::custom(format!( - "duplicate nested option: {option_name:?} in {name:?}" + "duplicate nested option: `{option_name}` in `{name}`" ))); } } @@ -159,20 +159,28 @@ impl Serialize for PressioCompressor { Ok(config) } - let options = { - let compressor = self.compressor.lock().map_err(serde::ser::Error::custom)?; - compressor - .get_options() - .map_err(serde::ser::Error::custom)? - }; + let compressor = self.compressor.lock().map_err(serde::ser::Error::custom)?; + let options = compressor + .get_options() + .map_err(serde::ser::Error::custom)?; + let metric_results = compressor + .get_metric_results() + .map_err(serde::ser::Error::custom)?; + let name = compressor.get_name().map_err(serde::ser::Error::custom)?; - PressioCompressorBorrowedFormat { + let result = PressioCompressorBorrowedFormat { compressor_id: self.compressor_id.as_str(), early_config: &self.early_config, compressor_config: &convert_from_pressio_options(options.iter())?, - name: self.name.as_deref(), + metric_results: &convert_from_pressio_options(metric_results.iter())?, + name: match name { + "" => Option::None, + name => Some(name), + }, } - .serialize(serializer) + .serialize(serializer); + std::mem::drop(compressor); + result } } @@ -182,6 +190,7 @@ impl<'de> Deserialize<'de> for PressioCompressor { fn convert_to_pressio_options( config: &BTreeMap, template: Option<&libpressio::PressioOptions>, + documentation: &libpressio::PressioOptions, ) -> Result { let mut options = libpressio::PressioOptions::new().map_err(serde::de::Error::custom)?; @@ -227,8 +236,15 @@ impl<'de> Deserialize<'de> for PressioCompressor { let Some(option_template) = template.get(&name).map_err(serde::de::Error::custom)? else { + let supported_options = template + .iter() + .filter_map(|(key, _value)| key) + .map(|x| format!("`{x}`")) + .collect::>() + .join(", "); + return Err(serde::de::Error::custom(format!( - "unknown compressor configuration option: {name:?}" + "unknown compressor configuration option: `{name}`, use one of {supported_options}" ))); }; @@ -239,11 +255,24 @@ impl<'de> Deserialize<'de> for PressioCompressor { if let Some(option) = option { options .set_with_cast( - name, + &name, option, libpressio::PressioConversionSafety::Special, ) - .map_err(serde::de::Error::custom)?; + .map_err(|err| { + let docs = match documentation.get(&name) { + Ok(Some(libpressio::PressioOption::string(Some(docs)))) => { + Some(docs) + } + _ => Option::None, + }; + + if let Some(docs) = docs { + serde::de::Error::custom(format_args!("{err} ({docs})")) + } else { + serde::de::Error::custom(err) + } + })?; } } else if let Some(option) = option { options @@ -258,6 +287,7 @@ impl<'de> Deserialize<'de> for PressioCompressor { // TODO: better error handling let format = PressioCompressorOwnedFormat::deserialize(deserializer)?; + std::mem::drop(format.metric_results); let mut pressio = libpressio::Pressio::new().map_err(serde::de::Error::custom)?; let mut compressor = pressio @@ -274,8 +304,7 @@ impl<'de> Deserialize<'de> for PressioCompressor { ); serde::de::Error::custom(format_args!( - "{}, choose one of: {}", - err.message, supported_compressors + "{err}, choose one of: {supported_compressors}" )) })?; @@ -285,14 +314,22 @@ impl<'de> Deserialize<'de> for PressioCompressor { .map_err(serde::de::Error::custom)?; } - let early_options = convert_to_pressio_options(&format.early_config, Option::None)?; + let documentation = compressor + .get_documentation() + .map_err(serde::de::Error::custom)?; + + let early_options = + convert_to_pressio_options(&format.early_config, Option::None, &documentation)?; compressor .set_options(&early_options) .map_err(serde::de::Error::custom)?; let options_template = compressor.get_options().map_err(serde::de::Error::custom)?; - let options = - convert_to_pressio_options(&format.compressor_config, Some(&options_template))?; + let options = convert_to_pressio_options( + &format.compressor_config, + Some(&options_template), + &documentation, + )?; compressor .set_options(&options) .map_err(serde::de::Error::custom)?; @@ -327,6 +364,9 @@ struct PressioCompressorOwnedFormat { /// Configuration for the compressor #[serde(default)] compressor_config: BTreeMap, + /// Results of the compressor metrics (output-only) + #[serde(default)] + metric_results: BTreeMap, /// Optional name for the compressor when used in hierarchical mode #[serde(default)] name: Option, @@ -338,13 +378,14 @@ struct PressioCompressorBorrowedFormat<'a> { /// The id of the compressor compressor_id: &'a str, /// Configuration for the structure of the compressor - #[serde(default)] early_config: &'a BTreeMap, /// Configuration for the compressor - #[serde(default)] compressor_config: &'a BTreeMap, + /// Results of the compressor metrics (output-only) + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + metric_results: &'a BTreeMap, /// Optional name for the compressor when used in hierarchical mode - #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] name: Option<&'a str>, } @@ -487,7 +528,7 @@ impl Codec for PressioCodec { libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); compressor - .compress(encoded, decompressed_data) + .decompress(encoded, decompressed_data) .map_err(|err| PressioCodecError::PressioDecodeFailed { source: PressioCodingError(err), }) @@ -554,7 +595,7 @@ impl Codec for PressioCodec { libpressio::PressioData::new_empty(decoded_dtype, decoded_shape); compressor - .compress(encoded, decompressed_data) + .decompress(encoded, decompressed_data) .map_err(|err| PressioCodecError::PressioDecodeFailed { source: PressioCodingError(err), }) From 15997f86be329e668bf2071ec80e6208368f79a8 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sat, 14 Mar 2026 13:40:18 +0200 Subject: [PATCH 37/55] support inline data nd arrays in the config --- Cargo.toml | 1 + codecs/pressio/Cargo.toml | 1 + codecs/pressio/src/lib.rs | 140 +++- codecs/pressio/tests/schema.json | 1331 ++++++++++++++++++++++++++++++ codecs/pressio/tests/schema.rs | 20 + 5 files changed, 1490 insertions(+), 3 deletions(-) create mode 100644 codecs/pressio/tests/schema.json create mode 100644 codecs/pressio/tests/schema.rs diff --git a/Cargo.toml b/Cargo.toml index d047975c4..632553493 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -118,6 +118,7 @@ schemars = { version = "1.0.3", default-features = false } scratch = { version = "1.0", default-features = false } semver = { version = "1.0.23", default-features = false } serde = { version = "1.0.218", default-features = false } +serde-ndim = { version = "=2.1.0", default-features = false } serde-transcode = { version = "1.1", default-features = false } serde_json = { version = "1.0.140", default-features = false } serde_repr = { version = "0.1.5", default-features = false } diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index e664605e6..5eb125e6f 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -20,6 +20,7 @@ ndarray = { workspace = true } numcodecs = { workspace = true } schemars = { workspace = true, features = ["derive", "preserve_order"] } serde = { workspace = true, features = ["std", "derive"] } +serde-ndim = { workspace = true, features = ["ndarray"] } thiserror = { workspace = true } [lints] diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 8fcd29d35..112897d53 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -23,7 +23,7 @@ use std::{ sync::Mutex, }; -use ndarray::{ArrayView, ArrayViewMut, CowArray, IxDyn}; +use ndarray::{Array, ArrayView, ArrayViewMut, CowArray, IxDyn}; use numcodecs::{ AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, StaticCodecConfig, StaticCodecVersion, @@ -74,6 +74,7 @@ impl Clone for PressioCompressor { } impl Serialize for PressioCompressor { + #[expect(clippy::too_many_lines)] fn serialize(&self, serializer: S) -> Result { fn convert_from_pressio_options( options: impl Iterator, Option)>, @@ -103,8 +104,21 @@ impl Serialize for PressioCompressor { libpressio::PressioOption::vec_string(Some(x)) => PressioOption::VecString(x), libpressio::PressioOption::dtype(Some(x)) => PressioOption::String(format!("{x}")), libpressio::PressioOption::thread_safety(Some(x)) => PressioOption::String(format!("{x}")), - libpressio::PressioOption::data(_) - | libpressio::PressioOption::user_ptr(_) + libpressio::PressioOption::data(Some(x)) => match x.clone_into_array() { + Option::None => continue, + Some(libpressio::PressioArray::Bool(x)) => PressioOption::DataBool(NdArray(x)), + Some(libpressio::PressioArray::Byte(x) | libpressio::PressioArray::U8(x)) => PressioOption::DataU8(NdArray(x)), + Some(libpressio::PressioArray::U16(x)) => PressioOption::DataU16(NdArray(x)), + Some(libpressio::PressioArray::U32(x)) => PressioOption::DataU32(NdArray(x)), + Some(libpressio::PressioArray::U64(x)) => PressioOption::DataU64(NdArray(x)), + Some(libpressio::PressioArray::I8(x)) => PressioOption::DataI8(NdArray(x)), + Some(libpressio::PressioArray::I16(x)) => PressioOption::DataI16(NdArray(x)), + Some(libpressio::PressioArray::I32(x)) => PressioOption::DataI32(NdArray(x)), + Some(libpressio::PressioArray::I64(x)) => PressioOption::DataI64(NdArray(x)), + Some(libpressio::PressioArray::F32(x)) => PressioOption::DataF32(NdArray(x)), + Some(libpressio::PressioArray::F64(x)) => PressioOption::DataF64(NdArray(x)), + }, + libpressio::PressioOption::user_ptr(_) | libpressio::PressioOption::unset | _ /* non-exhaustive */ => continue, }; @@ -218,6 +232,57 @@ impl<'de> Deserialize<'de> for PressioCompressor { PressioOption::VecString(x) => { Some(libpressio::PressioOption::vec_string(Some(x.clone()))) } + PressioOption::DataBool(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataU8(NdArray(x)) => Some(libpressio::PressioOption::data( + Some(libpressio::PressioData::new_copied(x)), + )), + PressioOption::DataU16(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataU32(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataU64(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataI8(NdArray(x)) => Some(libpressio::PressioOption::data( + Some(libpressio::PressioData::new_copied(x)), + )), + PressioOption::DataI16(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataI32(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataI64(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataF32(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataF64(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } PressioOption::Nested(entry) => { let mut nested_path = path.clone(); nested_path.push(key.clone()); @@ -408,9 +473,78 @@ pub enum PressioOption { F64(f64), String(String), VecString(Vec), + DataBool(NdArray), + DataU8(NdArray), + DataU16(NdArray), + DataU32(NdArray), + DataU64(NdArray), + DataI8(NdArray), + DataI16(NdArray), + DataI32(NdArray), + DataI64(NdArray), + DataF32(NdArray), + DataF64(NdArray), Nested(BTreeMap), } +#[derive(Clone)] +/// Pressio n-dimensional data array +pub struct NdArray(Array); + +impl std::fmt::Debug for NdArray { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + self.0.fmt(fmt) + } +} + +impl Serialize for NdArray { + fn serialize(&self, serializer: S) -> Result { + serde_ndim::serialize(&self.0, serializer) + } +} + +impl<'de, T: Deserialize<'de>> Deserialize<'de> for NdArray { + fn deserialize>(deserializer: D) -> Result { + serde_ndim::deserialize(deserializer).map(Self) + } +} + +impl JsonSchema for NdArray { + fn inline_schema() -> bool { + false + } + + fn schema_name() -> Cow<'static, str> { + Cow::Owned(format!("{}NdArray", std::any::type_name::())) + } + + fn schema_id() -> Cow<'static, str> { + Cow::Owned(format!( + "{}::NdArray<{}>", + module_path!(), + std::any::type_name::() + )) + } + + fn json_schema(generator: &mut SchemaGenerator) -> Schema { + let item = generator.subschema_for::(); + let nested = generator.subschema_for::(); + + json_schema!({ + "anyOf": [ + { + "type": "array", + "items": item, + }, + { + "type": "array", + "items": nested, + } + ] + }) + } +} + #[derive(Copy, Clone, Debug)] /// Equivalent of `Option::None` pub struct None; diff --git a/codecs/pressio/tests/schema.json b/codecs/pressio/tests/schema.json new file mode 100644 index 000000000..0cec2dfe5 --- /dev/null +++ b/codecs/pressio/tests/schema.json @@ -0,0 +1,1331 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "compressor_id": { + "type": "string", + "description": "The id of the compressor" + }, + "early_config": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + }, + { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + }, + { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + }, + { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + }, + { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + { + "type": "integer", + "format": "int32" + }, + { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + { + "type": "integer", + "format": "int64" + }, + { + "type": "number", + "format": "float" + }, + { + "type": "number", + "format": "double" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "boolean" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/boolNdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "float" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "double" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f64NdArray" + } + } + ] + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/PressioOption" + } + } + ], + "description": "Pressio option value" + }, + "description": "Configuration for the structure of the compressor", + "default": {} + }, + "compressor_config": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + }, + { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + }, + { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + }, + { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + }, + { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + { + "type": "integer", + "format": "int32" + }, + { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + { + "type": "integer", + "format": "int64" + }, + { + "type": "number", + "format": "float" + }, + { + "type": "number", + "format": "double" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "boolean" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/boolNdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "float" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "double" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f64NdArray" + } + } + ] + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/PressioOption" + } + } + ], + "description": "Pressio option value" + }, + "description": "Configuration for the compressor", + "default": {} + }, + "metric_results": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + }, + { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + }, + { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + }, + { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + }, + { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + { + "type": "integer", + "format": "int32" + }, + { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + { + "type": "integer", + "format": "int64" + }, + { + "type": "number", + "format": "float" + }, + { + "type": "number", + "format": "double" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "boolean" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/boolNdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "float" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "double" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f64NdArray" + } + } + ] + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/PressioOption" + } + } + ], + "description": "Pressio option value" + }, + "description": "Results of the compressor metrics (output-only)", + "default": {} + }, + "name": { + "type": [ + "string", + "null" + ], + "description": "Optional name for the compressor when used in hierarchical mode", + "default": null + }, + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "1.0.0" + } + }, + "required": [ + "compressor_id" + ], + "description": "Pressio codec which applies the identity function, i.e. passes through the\ninput unchanged during encoding and decoding.", + "title": "PressioCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "boolNdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "boolean" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/boolNdArray" + } + } + ] + }, + "u8NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u8NdArray" + } + } + ] + }, + "u16NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u16NdArray" + } + } + ] + }, + "u32NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u32NdArray" + } + } + ] + }, + "u64NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u64NdArray" + } + } + ] + }, + "i8NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i8NdArray" + } + } + ] + }, + "i16NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i16NdArray" + } + } + ] + }, + "i32NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i32NdArray" + } + } + ] + }, + "i64NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i64NdArray" + } + } + ] + }, + "f32NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "float" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f32NdArray" + } + } + ] + }, + "f64NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "double" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f64NdArray" + } + } + ] + }, + "PressioOption": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + }, + { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + }, + { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + }, + { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + }, + { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + { + "type": "integer", + "format": "int32" + }, + { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + { + "type": "integer", + "format": "int64" + }, + { + "type": "number", + "format": "float" + }, + { + "type": "number", + "format": "double" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "boolean" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/boolNdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "float" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "double" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f64NdArray" + } + } + ] + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/PressioOption" + } + } + ], + "description": "Pressio option value" + } + } +} \ No newline at end of file diff --git a/codecs/pressio/tests/schema.rs b/codecs/pressio/tests/schema.rs new file mode 100644 index 000000000..8945df79b --- /dev/null +++ b/codecs/pressio/tests/schema.rs @@ -0,0 +1,20 @@ +#![expect(missing_docs)] + +use ::{libpressio as _, ndarray as _, schemars as _, serde as _, serde_ndim as _, thiserror as _}; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_pressio::PressioCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + if schema != include_str!("schema.json") { + panic!("Pressio schema has changed\n===\n{schema}\n==="); + } +} From b87d0c47fc539ed6f699b1c7dac1b3309d1a7180 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sun, 15 Mar 2026 07:07:06 +0200 Subject: [PATCH 38/55] add linear quantizer test --- Cargo.toml | 2 +- codecs/pressio/Cargo.toml | 4 +++ codecs/pressio/src/lib.rs | 49 +++++++++++++++++++++++++++++++++- codecs/pressio/tests/schema.rs | 5 +++- 4 files changed, 57 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 632553493..db2618e5e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "d928ca7", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "1289c47", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index 5eb125e6f..074682eee 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -23,5 +23,9 @@ serde = { workspace = true, features = ["std", "derive"] } serde-ndim = { workspace = true, features = ["ndarray"] } thiserror = { workspace = true } +[dev-dependencies] +ndarray = { workspace = true, features = ["std"] } +serde_json = { workspace = true, features = ["std"] } + [lints] workspace = true diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 112897d53..13078ca77 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -100,7 +100,6 @@ impl Serialize for PressioCompressor { libpressio::PressioOption::float32(Some(x)) => PressioOption::F32(x), libpressio::PressioOption::float64(Some(x)) => PressioOption::F64(x), libpressio::PressioOption::string(Some(x)) => PressioOption::String(x), - // FIXME: seems to return strings as a single joined string libpressio::PressioOption::vec_string(Some(x)) => PressioOption::VecString(x), libpressio::PressioOption::dtype(Some(x)) => PressioOption::String(format!("{x}")), libpressio::PressioOption::thread_safety(Some(x)) => PressioOption::String(format!("{x}")), @@ -939,3 +938,51 @@ pub enum PressioCodecError { #[error(transparent)] /// Opaque error for when encoding or decoding with libpressio fails pub struct PressioCodingError(libpressio::PressioError); + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + use ndarray::Array1; + use serde_json::json; + + #[test] + fn linear_quantizer() { + let pressio = PressioCodec::deserialize(json!({ + "compressor_id": "linear_quantizer", + "early_config": { + "pressio:metric": "composite", + }, + "compressor_config": { + "pressio:abs": 10.0, + "pressio:metric": "composite", + "composite:plugins": ["printer", "size"], + } + })) + .unwrap(); + + let data = ndarray::linspace(0.0, 100.0, 50) + .collect::>() + .into_dyn(); + + let encoded = pressio + .encode(AnyCowArray::F64(CowArray::from(&data))) + .unwrap(); + + let decoded = pressio.decode(encoded.cow()); + assert!(matches!( + decoded, + Err(PressioCodecError::DecodeToArrayWithoutData) + )); + + let mut decoded = ndarray::Array::zeros(data.dim()); + pressio + .decode_into(encoded.view(), AnyArrayViewMut::F64(decoded.view_mut())) + .unwrap(); + + for (i, o) in data.iter().zip(decoded.iter()) { + assert!(((*i) - (*o)).abs() <= 10.0); + } + } +} diff --git a/codecs/pressio/tests/schema.rs b/codecs/pressio/tests/schema.rs index 8945df79b..ae8dde7ff 100644 --- a/codecs/pressio/tests/schema.rs +++ b/codecs/pressio/tests/schema.rs @@ -1,6 +1,9 @@ #![expect(missing_docs)] -use ::{libpressio as _, ndarray as _, schemars as _, serde as _, serde_ndim as _, thiserror as _}; +use ::{ + libpressio as _, ndarray as _, schemars as _, serde as _, serde_json as _, serde_ndim as _, + thiserror as _, +}; use numcodecs::{DynCodecType, StaticCodecType}; use numcodecs_pressio::PressioCodec; From 85da9474a685aa8a248a2f4a317b2e81cf3b843d Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 16 Mar 2026 01:05:53 +0200 Subject: [PATCH 39/55] fix pressio metrics results --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 153 ++++++++++++++++++++++++-------------- 2 files changed, 98 insertions(+), 57 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index db2618e5e..e88d3c0a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "1289c47", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5be5b1b", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 13078ca77..64400c9dc 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -20,7 +20,7 @@ use std::{ borrow::Cow, collections::{BTreeMap, btree_map::Entry}, - sync::Mutex, + sync::{Arc, Mutex, RwLock}, }; use ndarray::{Array, ArrayView, ArrayViewMut, CowArray, IxDyn}; @@ -47,28 +47,51 @@ pub struct PressioCodec { /// Pressio compressor pub struct PressioCompressor { + // get_config clones the compressor, but we want the config to include the + // compressor metrics + // so we make cheap shallow clones whenever possible and then later make + // the compressor unique with the clone-on-write `Arc::make_mut` + // we pinky-promise to only lock the inner `Mutex` for immutable access + // when we have read-only access, otherwise we can go through + // `Mutex::get_mut` + inner: RwLock>, +} + +impl Clone for PressioCompressor { + #[expect(clippy::unwrap_used)] + fn clone(&self) -> Self { + Self { + inner: RwLock::new(self.inner.read().unwrap().clone()), + } + } +} + +struct PressioCompressorInner { compressor: Mutex, compressor_id: String, early_config: BTreeMap, - name: Option, } -impl Clone for PressioCompressor { +impl Clone for PressioCompressorInner { #[expect(clippy::unwrap_used)] fn clone(&self) -> Self { let mut pressio = libpressio::Pressio::new().unwrap(); - let mut compressor = pressio.get_compressor(self.compressor_id.as_str()).unwrap(); - if let Some(name) = &self.name { - compressor.set_name(name).unwrap(); - } - let options = self.compressor.lock().unwrap().get_options().unwrap(); - compressor.set_options(&options).unwrap(); + let compressor = self.compressor.lock().unwrap(); + + let mut compressor_clone = pressio.get_compressor(self.compressor_id.as_str()).unwrap(); + compressor_clone + .set_name(compressor.get_name().unwrap()) + .unwrap(); + compressor_clone + .set_options(&compressor.get_options().unwrap()) + .unwrap(); + + std::mem::drop(compressor); Self { - compressor: Mutex::new(compressor), + compressor: Mutex::new(compressor_clone), compressor_id: self.compressor_id.clone(), early_config: self.early_config.clone(), - name: self.name.clone(), } } } @@ -172,7 +195,8 @@ impl Serialize for PressioCompressor { Ok(config) } - let compressor = self.compressor.lock().map_err(serde::ser::Error::custom)?; + let inner = self.inner.read().map_err(serde::ser::Error::custom)?; + let compressor = inner.compressor.lock().map_err(serde::ser::Error::custom)?; let options = compressor .get_options() .map_err(serde::ser::Error::custom)?; @@ -182,8 +206,8 @@ impl Serialize for PressioCompressor { let name = compressor.get_name().map_err(serde::ser::Error::custom)?; let result = PressioCompressorBorrowedFormat { - compressor_id: self.compressor_id.as_str(), - early_config: &self.early_config, + compressor_id: inner.compressor_id.as_str(), + early_config: &inner.early_config, compressor_config: &convert_from_pressio_options(options.iter())?, metric_results: &convert_from_pressio_options(metric_results.iter())?, name: match name { @@ -193,6 +217,7 @@ impl Serialize for PressioCompressor { } .serialize(serializer); std::mem::drop(compressor); + std::mem::drop(inner); result } } @@ -399,10 +424,11 @@ impl<'de> Deserialize<'de> for PressioCompressor { .map_err(serde::de::Error::custom)?; Ok(Self { - compressor: Mutex::new(compressor), - compressor_id: format.compressor_id, - early_config: format.early_config, - name: format.name, + inner: RwLock::new(Arc::new(PressioCompressorInner { + compressor: Mutex::new(compressor), + compressor_id: format.compressor_id, + early_config: format.early_config, + })), }) } } @@ -632,21 +658,25 @@ impl Codec for PressioCodec { } } - let Ok(mut compressor) = self.compressor.compressor.lock() else { - return Err(PressioCodecError::PressioPoisonedMutex); + let Ok(mut inner) = self.compressor.inner.write() else { + return Err(PressioCodecError::PressioPoisonedLock); + }; + + let Ok(compressor) = Arc::make_mut(&mut inner).compressor.get_mut() else { + return Err(PressioCodecError::PressioPoisonedLock); }; match data { - AnyCowArray::U8(data) => encode_typed(&mut compressor, data), - AnyCowArray::U16(data) => encode_typed(&mut compressor, data), - AnyCowArray::U32(data) => encode_typed(&mut compressor, data), - AnyCowArray::U64(data) => encode_typed(&mut compressor, data), - AnyCowArray::I8(data) => encode_typed(&mut compressor, data), - AnyCowArray::I16(data) => encode_typed(&mut compressor, data), - AnyCowArray::I32(data) => encode_typed(&mut compressor, data), - AnyCowArray::I64(data) => encode_typed(&mut compressor, data), - AnyCowArray::F32(data) => encode_typed(&mut compressor, data), - AnyCowArray::F64(data) => encode_typed(&mut compressor, data), + AnyCowArray::U8(data) => encode_typed(compressor, data), + AnyCowArray::U16(data) => encode_typed(compressor, data), + AnyCowArray::U32(data) => encode_typed(compressor, data), + AnyCowArray::U64(data) => encode_typed(compressor, data), + AnyCowArray::I8(data) => encode_typed(compressor, data), + AnyCowArray::I16(data) => encode_typed(compressor, data), + AnyCowArray::I32(data) => encode_typed(compressor, data), + AnyCowArray::I64(data) => encode_typed(compressor, data), + AnyCowArray::F32(data) => encode_typed(compressor, data), + AnyCowArray::F64(data) => encode_typed(compressor, data), data => Err(PressioCodecError::UnsupportedDtype(data.dtype())), } } @@ -692,21 +722,25 @@ impl Codec for PressioCodec { } } - let Ok(mut compressor) = self.compressor.compressor.lock() else { - return Err(PressioCodecError::PressioPoisonedMutex); + let Ok(mut inner) = self.compressor.inner.write() else { + return Err(PressioCodecError::PressioPoisonedLock); + }; + + let Ok(compressor) = Arc::make_mut(&mut inner).compressor.get_mut() else { + return Err(PressioCodecError::PressioPoisonedLock); }; match encoded { - AnyCowArray::U8(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::U16(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::U32(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::U64(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::I8(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::I16(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::I32(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::I64(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::F32(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::F64(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::U8(encoded) => decode_typed(compressor, encoded), + AnyCowArray::U16(encoded) => decode_typed(compressor, encoded), + AnyCowArray::U32(encoded) => decode_typed(compressor, encoded), + AnyCowArray::U64(encoded) => decode_typed(compressor, encoded), + AnyCowArray::I8(encoded) => decode_typed(compressor, encoded), + AnyCowArray::I16(encoded) => decode_typed(compressor, encoded), + AnyCowArray::I32(encoded) => decode_typed(compressor, encoded), + AnyCowArray::I64(encoded) => decode_typed(compressor, encoded), + AnyCowArray::F32(encoded) => decode_typed(compressor, encoded), + AnyCowArray::F64(encoded) => decode_typed(compressor, encoded), encoded => Err(PressioCodecError::UnsupportedDtype(encoded.dtype())), } } @@ -803,8 +837,12 @@ impl Codec for PressioCodec { Ok(()) } - let Ok(mut compressor) = self.compressor.compressor.lock() else { - return Err(PressioCodecError::PressioPoisonedMutex); + let Ok(mut inner) = self.compressor.inner.write() else { + return Err(PressioCodecError::PressioPoisonedLock); + }; + + let Ok(compressor) = Arc::make_mut(&mut inner).compressor.get_mut() else { + return Err(PressioCodecError::PressioPoisonedLock); }; let decoded_dtype = match decoded.dtype() { @@ -824,34 +862,34 @@ impl Codec for PressioCodec { let decompressed_data = match encoded { AnyArrayView::U8(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::U16(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::U32(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::U64(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::I8(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::I16(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::I32(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::I64(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::F32(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::F64(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } encoded => return Err(PressioCodecError::UnsupportedDtype(encoded.dtype())), }?; @@ -894,7 +932,7 @@ pub enum PressioCodecError { UnsupportedDtype(AnyArrayDType), /// [`PressioCodec`] lock was poisoned #[error("Pressio lock was poisoned")] - PressioPoisonedMutex, + PressioPoisonedLock, /// [`PressioCodec`] failed to encode the data #[error("Pressio failed to encode the data")] PressioEncodeFailed { @@ -957,7 +995,7 @@ mod tests { "compressor_config": { "pressio:abs": 10.0, "pressio:metric": "composite", - "composite:plugins": ["printer", "size"], + "composite:plugins": ["printer", "size", "time"], } })) .unwrap(); @@ -984,5 +1022,8 @@ mod tests { for (i, o) in data.iter().zip(decoded.iter()) { assert!(((*i) - (*o)).abs() <= 10.0); } + + let config = serde_json::to_string(&pressio.get_config()).unwrap(); + assert!(config.contains("\"size:compressed_size\":400")); } } From 45da30cf78316c7d2e5abda11b7590181287a9fc Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 16 Mar 2026 10:59:48 +0200 Subject: [PATCH 40/55] add support for the libpressio bzip2 compressor --- Cargo.toml | 2 +- codecs/pressio/Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 41 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e88d3c0a6..d3a0678b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5be5b1b", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "4508eb7", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index 074682eee..1a4966743 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -15,7 +15,7 @@ keywords = ["libpressio", "numcodecs", "compression", "encoding"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -libpressio = { workspace = true } +libpressio = { workspace = true, features = ["bzip2"] } ndarray = { workspace = true } numcodecs = { workspace = true } schemars = { workspace = true, features = ["derive", "preserve_order"] } diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 64400c9dc..eb6ccc0fa 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -986,7 +986,7 @@ mod tests { use serde_json::json; #[test] - fn linear_quantizer() { + fn linear_quantizer_noop() { let pressio = PressioCodec::deserialize(json!({ "compressor_id": "linear_quantizer", "early_config": { @@ -1026,4 +1026,43 @@ mod tests { let config = serde_json::to_string(&pressio.get_config()).unwrap(); assert!(config.contains("\"size:compressed_size\":400")); } + + #[test] + fn linear_quantizer_bzip2() { + let pressio = PressioCodec::deserialize(json!({ + "compressor_id": "linear_quantizer", + "early_config": { + "linear_quantizer:compressor": "bzip2", + }, + "compressor_config": { + "pressio:abs": 10.0, + "pressio:lossless": 9, + "pressio:metric": "size", + } + })) + .unwrap(); + + let data = ndarray::linspace(0.0, 100.0, 50) + .collect::>() + .into_dyn(); + + let encoded = pressio + .encode(AnyCowArray::F64(CowArray::from(&data))) + .unwrap(); + + let decoded = pressio.decode(encoded.cow()); + assert!(decoded.is_err()); + + let mut decoded = ndarray::Array::zeros(data.dim()); + pressio + .decode_into(encoded.view(), AnyArrayViewMut::F64(decoded.view_mut())) + .unwrap(); + + for (i, o) in data.iter().zip(decoded.iter()) { + assert!(((*i) - (*o)).abs() <= 10.0); + } + + let config = serde_json::to_string(&pressio.get_config()).unwrap(); + assert!(config.contains("\"size:compressed_size\":63")); + } } From 506e19fb887434bbc25c3b343fc485fe0749067a Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Tue, 17 Mar 2026 09:17:44 +0200 Subject: [PATCH 41/55] Try out Lua support for libpressio --- Cargo.toml | 2 +- codecs/pressio/Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 52 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d3a0678b8..1aca865f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "4508eb7", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "22f085e", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index 1a4966743..ffe83260b 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -15,7 +15,7 @@ keywords = ["libpressio", "numcodecs", "compression", "encoding"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -libpressio = { workspace = true, features = ["bzip2"] } +libpressio = { workspace = true, features = ["bzip2", "lua"] } ndarray = { workspace = true } numcodecs = { workspace = true } schemars = { workspace = true, features = ["derive", "preserve_order"] } diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index eb6ccc0fa..8ccf0d0b3 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -1065,4 +1065,56 @@ mod tests { let config = serde_json::to_string(&pressio.get_config()).unwrap(); assert!(config.contains("\"size:compressed_size\":63")); } + + #[test] + fn lua_metrics() { + let pressio = PressioCodec::deserialize(json!({ + "compressor_id": "noop", + "early_config": { + "pressio:metric": "composite", + }, + "compressor_config": { + "pressio:metric": "composite", + "composite:plugins": ["size"], + "composite:scripts": [ + "return \"objective\", 1.2", + "return \"objective2\", metrics[\"size:compression_ratio\"] * 4.2", + ] + } + })) + .unwrap(); + + let config = serde_json::to_string(&pressio.get_config()).unwrap(); + assert!(!config.contains("\"size:compression_ratio\"")); + assert!(config.contains("\"composite:objective\":1.2")); + assert!(!config.contains("\"composite:objective2\"")); + + let data = ndarray::linspace(0.0, 100.0, 50) + .collect::>() + .into_dyn(); + + let encoded = pressio + .encode(AnyCowArray::F64(CowArray::from(&data))) + .unwrap(); + + let decoded = pressio.decode(encoded.cow()); + assert!(matches!( + decoded, + Err(PressioCodecError::DecodeToArrayWithoutData) + )); + + let mut decoded = ndarray::Array::zeros(data.dim()); + pressio + .decode_into(encoded.view(), AnyArrayViewMut::F64(decoded.view_mut())) + .unwrap(); + + for (i, o) in data.iter().zip(decoded.iter()) { + assert!(i.to_bits() == o.to_bits()); + } + + let config = serde_json::to_string(&pressio.get_config()).unwrap(); + assert!(config.contains("\"size:compression_ratio\":1.0")); + assert!(config.contains("\"composite:objective\":1.2")); + assert!(config.contains("\"composite:objective2\":4.2")); + } } From 5922a96406712ec0901132367a3d10c583519667 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sun, 22 Mar 2026 08:24:29 +0200 Subject: [PATCH 42/55] update libpressio-rs --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 1aca865f1..dd860554e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "22f085e", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "255ed51", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From 697925a7e637903fcb49afa1ff72ecb5735cc3ae Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 26 Mar 2026 14:35:33 +0200 Subject: [PATCH 43/55] allow using non-sendable pressio compressors using fragile --- Cargo.toml | 3 ++- codecs/pressio/Cargo.toml | 1 + codecs/pressio/src/lib.rs | 56 +++++++++++++++++++++++++++++++++++---- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dd860554e..658a5243b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,10 +93,11 @@ clap = { version = "4.5", default-features = false } convert_case = { version = "0.8", default-features = false } ebcc = { version = "0.1", default-features = false } format_serde_error = { version = "0.3", default-features = false } +fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "255ed51", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "30545dd", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index ffe83260b..87a49a490 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -15,6 +15,7 @@ keywords = ["libpressio", "numcodecs", "compression", "encoding"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +fragile = { workspace = true } libpressio = { workspace = true, features = ["bzip2", "lua"] } ndarray = { workspace = true } numcodecs = { workspace = true } diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 8ccf0d0b3..2da4148a3 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -23,6 +23,7 @@ use std::{ sync::{Arc, Mutex, RwLock}, }; +use fragile::Fragile; use ndarray::{Array, ArrayView, ArrayViewMut, CowArray, IxDyn}; use numcodecs::{ AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray, @@ -67,7 +68,7 @@ impl Clone for PressioCompressor { } struct PressioCompressorInner { - compressor: Mutex, + compressor: Mutex, compressor_id: String, early_config: BTreeMap, } @@ -76,7 +77,8 @@ impl Clone for PressioCompressorInner { #[expect(clippy::unwrap_used)] fn clone(&self) -> Self { let mut pressio = libpressio::Pressio::new().unwrap(); - let compressor = self.compressor.lock().unwrap(); + let compressor_guard = self.compressor.lock().unwrap(); + let compressor = compressor_guard.try_get().unwrap(); let mut compressor_clone = pressio.get_compressor(self.compressor_id.as_str()).unwrap(); compressor_clone @@ -86,7 +88,12 @@ impl Clone for PressioCompressorInner { .set_options(&compressor.get_options().unwrap()) .unwrap(); - std::mem::drop(compressor); + std::mem::drop(compressor_guard); + + let compressor_clone = match compressor_clone.try_into_sendable() { + Ok(compressor) => PressioCompressorSendable::Sendable(compressor), + Err((compressor, _err)) => PressioCompressorSendable::Fragile(Fragile::new(compressor)), + }; Self { compressor: Mutex::new(compressor_clone), @@ -96,6 +103,31 @@ impl Clone for PressioCompressorInner { } } +enum PressioCompressorSendable { + Sendable(libpressio::PressioSendableCompressor), + Fragile(Fragile), +} + +impl PressioCompressorSendable { + fn try_get(&self) -> Result<&libpressio::PressioCompressor, PressioCodecError> { + match self { + Self::Sendable(compressor) => Ok(compressor), + Self::Fragile(compressor) => compressor + .try_get() + .map_err(|_| PressioCodecError::PressioNonThreadsafeSend), + } + } + + fn try_get_mut(&mut self) -> Result<&mut libpressio::PressioCompressor, PressioCodecError> { + match self { + Self::Sendable(compressor) => Ok(compressor), + Self::Fragile(compressor) => compressor + .try_get_mut() + .map_err(|_| PressioCodecError::PressioNonThreadsafeSend), + } + } +} + impl Serialize for PressioCompressor { #[expect(clippy::too_many_lines)] fn serialize(&self, serializer: S) -> Result { @@ -196,7 +228,10 @@ impl Serialize for PressioCompressor { } let inner = self.inner.read().map_err(serde::ser::Error::custom)?; - let compressor = inner.compressor.lock().map_err(serde::ser::Error::custom)?; + let compressor_guard = inner.compressor.lock().map_err(serde::ser::Error::custom)?; + let compressor = compressor_guard + .try_get() + .map_err(serde::ser::Error::custom)?; let options = compressor .get_options() .map_err(serde::ser::Error::custom)?; @@ -216,7 +251,7 @@ impl Serialize for PressioCompressor { }, } .serialize(serializer); - std::mem::drop(compressor); + std::mem::drop(compressor_guard); std::mem::drop(inner); result } @@ -423,6 +458,11 @@ impl<'de> Deserialize<'de> for PressioCompressor { .set_options(&options) .map_err(serde::de::Error::custom)?; + let compressor = match compressor.try_into_sendable() { + Ok(compressor) => PressioCompressorSendable::Sendable(compressor), + Err((compressor, _err)) => PressioCompressorSendable::Fragile(Fragile::new(compressor)), + }; + Ok(Self { inner: RwLock::new(Arc::new(PressioCompressorInner { compressor: Mutex::new(compressor), @@ -665,6 +705,7 @@ impl Codec for PressioCodec { let Ok(compressor) = Arc::make_mut(&mut inner).compressor.get_mut() else { return Err(PressioCodecError::PressioPoisonedLock); }; + let compressor = compressor.try_get_mut()?; match data { AnyCowArray::U8(data) => encode_typed(compressor, data), @@ -729,6 +770,7 @@ impl Codec for PressioCodec { let Ok(compressor) = Arc::make_mut(&mut inner).compressor.get_mut() else { return Err(PressioCodecError::PressioPoisonedLock); }; + let compressor = compressor.try_get_mut()?; match encoded { AnyCowArray::U8(encoded) => decode_typed(compressor, encoded), @@ -844,6 +886,7 @@ impl Codec for PressioCodec { let Ok(compressor) = Arc::make_mut(&mut inner).compressor.get_mut() else { return Err(PressioCodecError::PressioPoisonedLock); }; + let compressor = compressor.try_get_mut()?; let decoded_dtype = match decoded.dtype() { AnyArrayDType::U8 => libpressio::PressioDtype::U8, @@ -933,6 +976,9 @@ pub enum PressioCodecError { /// [`PressioCodec`] lock was poisoned #[error("Pressio lock was poisoned")] PressioPoisonedLock, + /// [`PressioCodec`] was used on a different thread with a non-threadsafe compressor + #[error("Pressio was used on a different thread with a non-threadsafe compressor")] + PressioNonThreadsafeSend, /// [`PressioCodec`] failed to encode the data #[error("Pressio failed to encode the data")] PressioEncodeFailed { From 0c2993942b0186b473511e06c636981440168486 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 22 Apr 2026 06:39:53 +0300 Subject: [PATCH 44/55] Update libpressio-rs --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 658a5243b..c683b7c9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,7 +97,7 @@ fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "30545dd", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "39d2fcb", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From 1082c6cb4948cdf98f5565a9e4c83dac8655d313 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 22 Apr 2026 07:02:51 +0300 Subject: [PATCH 45/55] Bump MSRV to 1.88 --- .github/workflows/ci.yml | 4 ++-- Cargo.toml | 4 ++-- README.md | 2 +- codecs/asinh/README.md | 2 +- codecs/asinh/src/lib.rs | 2 +- codecs/bit-round/README.md | 2 +- codecs/bit-round/src/lib.rs | 2 +- codecs/ebcc/README.md | 2 +- codecs/ebcc/src/lib.rs | 2 +- codecs/fixed-offset-scale/README.md | 2 +- codecs/fixed-offset-scale/src/lib.rs | 2 +- codecs/fourier-network/Cargo.toml | 2 +- codecs/fourier-network/README.md | 2 +- codecs/fourier-network/src/lib.rs | 4 ++-- codecs/identity/README.md | 2 +- codecs/identity/src/lib.rs | 2 +- codecs/jpeg2000/README.md | 2 +- codecs/jpeg2000/src/lib.rs | 2 +- codecs/lc/README.md | 2 +- codecs/lc/src/lib.rs | 2 +- codecs/linear-quantize/README.md | 2 +- codecs/linear-quantize/src/lib.rs | 2 +- codecs/log/README.md | 2 +- codecs/log/src/lib.rs | 2 +- codecs/pco/README.md | 2 +- codecs/pco/src/lib.rs | 2 +- codecs/qpet-sperr/README.md | 2 +- codecs/qpet-sperr/src/lib.rs | 2 +- codecs/random-projection/README.md | 2 +- codecs/random-projection/src/lib.rs | 2 +- codecs/reinterpret/README.md | 2 +- codecs/reinterpret/src/lib.rs | 2 +- codecs/round/README.md | 2 +- codecs/round/src/lib.rs | 2 +- codecs/sperr/README.md | 2 +- codecs/sperr/src/lib.rs | 2 +- codecs/stochastic-rounding/README.md | 2 +- codecs/stochastic-rounding/src/lib.rs | 2 +- codecs/swizzle-reshape/README.md | 2 +- codecs/swizzle-reshape/src/lib.rs | 2 +- codecs/sz3/README.md | 2 +- codecs/sz3/src/lib.rs | 2 +- codecs/tthresh/README.md | 2 +- codecs/tthresh/src/lib.rs | 2 +- codecs/uniform-noise/README.md | 2 +- codecs/uniform-noise/src/lib.rs | 2 +- codecs/zfp-classic/README.md | 2 +- codecs/zfp-classic/src/lib.rs | 2 +- codecs/zfp/README.md | 2 +- codecs/zfp/src/lib.rs | 2 +- codecs/zlib/README.md | 2 +- codecs/zlib/src/lib.rs | 2 +- codecs/zstd/README.md | 2 +- codecs/zstd/src/lib.rs | 2 +- crates/numcodecs-python/README.md | 2 +- crates/numcodecs-python/src/lib.rs | 2 +- crates/numcodecs-wasm-builder/README.md | 2 +- .../buildenv/flake.lock | 20 +++++++++---------- .../buildenv/rust-toolchain | 2 +- crates/numcodecs-wasm-guest/README.md | 2 +- crates/numcodecs-wasm-guest/src/lib.rs | 2 +- .../README.md | 2 +- .../src/lib.rs | 2 +- crates/numcodecs-wasm-host/README.md | 2 +- crates/numcodecs-wasm-host/src/lib.rs | 2 +- crates/numcodecs-wasm-logging/README.md | 2 +- crates/numcodecs-wasm-logging/src/lib.rs | 2 +- crates/numcodecs/README.md | 2 +- crates/numcodecs/src/lib.rs | 2 +- 69 files changed, 81 insertions(+), 81 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 195cdbb63..f76fcdcf3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - rust: ["1.87", stable, nightly] + rust: ["1.88", stable, nightly] lock: ["Cargo.lock", "Cargo.lock.min"] runs-on: ${{ matrix.os }} needs: lock @@ -178,7 +178,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - rust: ["1.87", stable] + rust: ["1.88", stable] lock: ["Cargo.lock", "Cargo.lock.min"] runs-on: ${{ matrix.os }} needs: lock diff --git a/Cargo.toml b/Cargo.toml index c683b7c9b..512b5acba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,7 @@ edition = "2024" authors = ["Juniper Tyree "] repository = "https://github.com/juntyr/numcodecs-rs" license = "MPL-2.0" -rust-version = "1.87" +rust-version = "1.88" [workspace.dependencies] # workspace-internal numcodecs crates @@ -97,7 +97,7 @@ fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "39d2fcb", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "edf7511", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/README.md b/README.md index e5d4934b4..a103393eb 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs diff --git a/codecs/asinh/README.md b/codecs/asinh/README.md index 6db74d008..14cd41347 100644 --- a/codecs/asinh/README.md +++ b/codecs/asinh/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-asinh diff --git a/codecs/asinh/src/lib.rs b/codecs/asinh/src/lib.rs index 9c1baf762..d685145a5 100644 --- a/codecs/asinh/src/lib.rs +++ b/codecs/asinh/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-asinh diff --git a/codecs/bit-round/README.md b/codecs/bit-round/README.md index b07b2be3f..fca5fcd4a 100644 --- a/codecs/bit-round/README.md +++ b/codecs/bit-round/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-bit-round diff --git a/codecs/bit-round/src/lib.rs b/codecs/bit-round/src/lib.rs index 5b9fa2364..e562fc2cf 100644 --- a/codecs/bit-round/src/lib.rs +++ b/codecs/bit-round/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-bit-round diff --git a/codecs/ebcc/README.md b/codecs/ebcc/README.md index 1a72e510c..5678b9f94 100644 --- a/codecs/ebcc/README.md +++ b/codecs/ebcc/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-ebcc diff --git a/codecs/ebcc/src/lib.rs b/codecs/ebcc/src/lib.rs index 2068f723d..fb35b5704 100644 --- a/codecs/ebcc/src/lib.rs +++ b/codecs/ebcc/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-ebcc diff --git a/codecs/fixed-offset-scale/README.md b/codecs/fixed-offset-scale/README.md index 8ab537fcc..73c0f3387 100644 --- a/codecs/fixed-offset-scale/README.md +++ b/codecs/fixed-offset-scale/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-fixed-offset-scale diff --git a/codecs/fixed-offset-scale/src/lib.rs b/codecs/fixed-offset-scale/src/lib.rs index 920c0f4e8..555aea69b 100644 --- a/codecs/fixed-offset-scale/src/lib.rs +++ b/codecs/fixed-offset-scale/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-fixed-offset-scale diff --git a/codecs/fourier-network/Cargo.toml b/codecs/fourier-network/Cargo.toml index 202c97974..8f2a6a775 100644 --- a/codecs/fourier-network/Cargo.toml +++ b/codecs/fourier-network/Cargo.toml @@ -18,7 +18,7 @@ keywords = ["fourier", "network", "numcodecs", "compression", "encoding"] burn = { workspace = true, features = ["std", "autodiff", "ndarray"] } itertools = { workspace = true, features = ["use_alloc"] } log = { workspace = true } -# FIXME: bytemuck 1.24 fails to compile on 1.87 +# FIXME: bytemuck 1.24 fails to compile on 1.88 bytemuck = { version = "=1.23.2", default-features = false } ndarray = { workspace = true, features = ["std"] } numcodecs = { workspace = true } diff --git a/codecs/fourier-network/README.md b/codecs/fourier-network/README.md index ff45edd5b..76cfdb5e6 100644 --- a/codecs/fourier-network/README.md +++ b/codecs/fourier-network/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-fourier-network diff --git a/codecs/fourier-network/src/lib.rs b/codecs/fourier-network/src/lib.rs index 10dddef9f..d2d1121f4 100644 --- a/codecs/fourier-network/src/lib.rs +++ b/codecs/fourier-network/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-fourier-network @@ -46,7 +46,7 @@ use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; -// FIXME: bytemuck 1.24 fails to compile on 1.87 +// FIXME: bytemuck 1.24 fails to compile on 1.88 use ::bytemuck as _; // FIXME: burn-common -> cubecl-common brings in wasm-bindgen diff --git a/codecs/identity/README.md b/codecs/identity/README.md index e7c0e3a88..a009d45c4 100644 --- a/codecs/identity/README.md +++ b/codecs/identity/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-identity diff --git a/codecs/identity/src/lib.rs b/codecs/identity/src/lib.rs index 92aa2d649..4535da6dc 100644 --- a/codecs/identity/src/lib.rs +++ b/codecs/identity/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-identity diff --git a/codecs/jpeg2000/README.md b/codecs/jpeg2000/README.md index 8bb11ff84..17c21fbc7 100644 --- a/codecs/jpeg2000/README.md +++ b/codecs/jpeg2000/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-jpeg2000 diff --git a/codecs/jpeg2000/src/lib.rs b/codecs/jpeg2000/src/lib.rs index 8f3211db7..6f00b7b02 100644 --- a/codecs/jpeg2000/src/lib.rs +++ b/codecs/jpeg2000/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-jpeg2000 diff --git a/codecs/lc/README.md b/codecs/lc/README.md index 3511eeaf2..c3c883a8c 100644 --- a/codecs/lc/README.md +++ b/codecs/lc/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-lc diff --git a/codecs/lc/src/lib.rs b/codecs/lc/src/lib.rs index 6c4ba3116..10e96fed4 100644 --- a/codecs/lc/src/lib.rs +++ b/codecs/lc/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-lc diff --git a/codecs/linear-quantize/README.md b/codecs/linear-quantize/README.md index c93b302c3..341c0ef2f 100644 --- a/codecs/linear-quantize/README.md +++ b/codecs/linear-quantize/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-linear-quantize diff --git a/codecs/linear-quantize/src/lib.rs b/codecs/linear-quantize/src/lib.rs index 1bb5a24c8..7803aecac 100644 --- a/codecs/linear-quantize/src/lib.rs +++ b/codecs/linear-quantize/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-linear-quantize diff --git a/codecs/log/README.md b/codecs/log/README.md index 8725ba54d..90a1d758c 100644 --- a/codecs/log/README.md +++ b/codecs/log/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-log diff --git a/codecs/log/src/lib.rs b/codecs/log/src/lib.rs index 46460daed..7061b4d5a 100644 --- a/codecs/log/src/lib.rs +++ b/codecs/log/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-log diff --git a/codecs/pco/README.md b/codecs/pco/README.md index faf730df4..e8ffc68c0 100644 --- a/codecs/pco/README.md +++ b/codecs/pco/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-pco diff --git a/codecs/pco/src/lib.rs b/codecs/pco/src/lib.rs index 5a9575b28..35d7d98ae 100644 --- a/codecs/pco/src/lib.rs +++ b/codecs/pco/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-pco diff --git a/codecs/qpet-sperr/README.md b/codecs/qpet-sperr/README.md index 8a317c0d3..058ba764d 100644 --- a/codecs/qpet-sperr/README.md +++ b/codecs/qpet-sperr/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-qpet-sperr diff --git a/codecs/qpet-sperr/src/lib.rs b/codecs/qpet-sperr/src/lib.rs index 929b8a645..445bbe24a 100644 --- a/codecs/qpet-sperr/src/lib.rs +++ b/codecs/qpet-sperr/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-qpet-sperr diff --git a/codecs/random-projection/README.md b/codecs/random-projection/README.md index 1bad2c39d..43958665d 100644 --- a/codecs/random-projection/README.md +++ b/codecs/random-projection/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-random-projection diff --git a/codecs/random-projection/src/lib.rs b/codecs/random-projection/src/lib.rs index f30c81b59..4510758bc 100644 --- a/codecs/random-projection/src/lib.rs +++ b/codecs/random-projection/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-random-projection diff --git a/codecs/reinterpret/README.md b/codecs/reinterpret/README.md index a5329096a..8e05d6bf4 100644 --- a/codecs/reinterpret/README.md +++ b/codecs/reinterpret/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-reinterpret diff --git a/codecs/reinterpret/src/lib.rs b/codecs/reinterpret/src/lib.rs index 638415d53..a39d74735 100644 --- a/codecs/reinterpret/src/lib.rs +++ b/codecs/reinterpret/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-reinterpret diff --git a/codecs/round/README.md b/codecs/round/README.md index 662671c3c..0d2fc4cd8 100644 --- a/codecs/round/README.md +++ b/codecs/round/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-round diff --git a/codecs/round/src/lib.rs b/codecs/round/src/lib.rs index 549611d39..aed61938f 100644 --- a/codecs/round/src/lib.rs +++ b/codecs/round/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-round diff --git a/codecs/sperr/README.md b/codecs/sperr/README.md index 4fbeb4542..7384e6c95 100644 --- a/codecs/sperr/README.md +++ b/codecs/sperr/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-sperr diff --git a/codecs/sperr/src/lib.rs b/codecs/sperr/src/lib.rs index 7b6435c8f..8195807c5 100644 --- a/codecs/sperr/src/lib.rs +++ b/codecs/sperr/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-sperr diff --git a/codecs/stochastic-rounding/README.md b/codecs/stochastic-rounding/README.md index 747fb4b06..9c351028f 100644 --- a/codecs/stochastic-rounding/README.md +++ b/codecs/stochastic-rounding/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-stochastic-rounding diff --git a/codecs/stochastic-rounding/src/lib.rs b/codecs/stochastic-rounding/src/lib.rs index 497f3234b..475c7e5ca 100644 --- a/codecs/stochastic-rounding/src/lib.rs +++ b/codecs/stochastic-rounding/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-stochastic-rounding diff --git a/codecs/swizzle-reshape/README.md b/codecs/swizzle-reshape/README.md index b276e8dfa..711672cff 100644 --- a/codecs/swizzle-reshape/README.md +++ b/codecs/swizzle-reshape/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-swizzle-reshape diff --git a/codecs/swizzle-reshape/src/lib.rs b/codecs/swizzle-reshape/src/lib.rs index f3bc3fa21..850d4c4d9 100644 --- a/codecs/swizzle-reshape/src/lib.rs +++ b/codecs/swizzle-reshape/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-swizzle-reshape diff --git a/codecs/sz3/README.md b/codecs/sz3/README.md index 7773e5f90..81d5c224f 100644 --- a/codecs/sz3/README.md +++ b/codecs/sz3/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-sz3 diff --git a/codecs/sz3/src/lib.rs b/codecs/sz3/src/lib.rs index 444f45877..b783e5dc7 100644 --- a/codecs/sz3/src/lib.rs +++ b/codecs/sz3/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-sz3 diff --git a/codecs/tthresh/README.md b/codecs/tthresh/README.md index 0c6e8d05e..ef0e787c3 100644 --- a/codecs/tthresh/README.md +++ b/codecs/tthresh/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-tthresh diff --git a/codecs/tthresh/src/lib.rs b/codecs/tthresh/src/lib.rs index 72e8d782b..0eceb5de3 100644 --- a/codecs/tthresh/src/lib.rs +++ b/codecs/tthresh/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-tthresh diff --git a/codecs/uniform-noise/README.md b/codecs/uniform-noise/README.md index 24392d5de..e022b51aa 100644 --- a/codecs/uniform-noise/README.md +++ b/codecs/uniform-noise/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-uniform-noise diff --git a/codecs/uniform-noise/src/lib.rs b/codecs/uniform-noise/src/lib.rs index ac1e8436d..cccedf3fa 100644 --- a/codecs/uniform-noise/src/lib.rs +++ b/codecs/uniform-noise/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-uniform-noise diff --git a/codecs/zfp-classic/README.md b/codecs/zfp-classic/README.md index 9f032069e..b553bbae0 100644 --- a/codecs/zfp-classic/README.md +++ b/codecs/zfp-classic/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-zfp-classic diff --git a/codecs/zfp-classic/src/lib.rs b/codecs/zfp-classic/src/lib.rs index f008a6cc9..47522be98 100644 --- a/codecs/zfp-classic/src/lib.rs +++ b/codecs/zfp-classic/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-zfp-classic diff --git a/codecs/zfp/README.md b/codecs/zfp/README.md index 3537c137a..0c358801a 100644 --- a/codecs/zfp/README.md +++ b/codecs/zfp/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-zfp diff --git a/codecs/zfp/src/lib.rs b/codecs/zfp/src/lib.rs index 3c16e4f87..cda53b053 100644 --- a/codecs/zfp/src/lib.rs +++ b/codecs/zfp/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-zfp diff --git a/codecs/zlib/README.md b/codecs/zlib/README.md index 37ce9b5aa..63991b516 100644 --- a/codecs/zlib/README.md +++ b/codecs/zlib/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-zlib diff --git a/codecs/zlib/src/lib.rs b/codecs/zlib/src/lib.rs index ea93ab42b..69bc3d438 100644 --- a/codecs/zlib/src/lib.rs +++ b/codecs/zlib/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-zlib diff --git a/codecs/zstd/README.md b/codecs/zstd/README.md index d3da36c92..3f437cdc5 100644 --- a/codecs/zstd/README.md +++ b/codecs/zstd/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-zstd diff --git a/codecs/zstd/src/lib.rs b/codecs/zstd/src/lib.rs index fcd3cde55..1744666a3 100644 --- a/codecs/zstd/src/lib.rs +++ b/codecs/zstd/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-zstd diff --git a/crates/numcodecs-python/README.md b/crates/numcodecs-python/README.md index 6dd4b22f0..6f762507f 100644 --- a/crates/numcodecs-python/README.md +++ b/crates/numcodecs-python/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-python diff --git a/crates/numcodecs-python/src/lib.rs b/crates/numcodecs-python/src/lib.rs index 9817aeddf..4635b05ce 100644 --- a/crates/numcodecs-python/src/lib.rs +++ b/crates/numcodecs-python/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-python diff --git a/crates/numcodecs-wasm-builder/README.md b/crates/numcodecs-wasm-builder/README.md index 7b625b2f9..86d608d63 100644 --- a/crates/numcodecs-wasm-builder/README.md +++ b/crates/numcodecs-wasm-builder/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-builder diff --git a/crates/numcodecs-wasm-builder/buildenv/flake.lock b/crates/numcodecs-wasm-builder/buildenv/flake.lock index e32be7dbf..d11c7d73e 100644 --- a/crates/numcodecs-wasm-builder/buildenv/flake.lock +++ b/crates/numcodecs-wasm-builder/buildenv/flake.lock @@ -2,12 +2,12 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1758346548, - "narHash": "sha256-afXE7AJ7MY6wY1pg/Y6UPHNYPy5GtUKeBkrZZ/gC71E=", - "rev": "b2a3852bd078e68dd2b3dfa8c00c67af1f0a7d20", - "revCount": 810175, + "lastModified": 1776560675, + "narHash": "sha256-p68udKWWh7+V4ZPpcMDq0gTHWNZJnr4JPI+kHPPE40o=", + "rev": "e07580dae39738e46609eaab8b154de2488133ce", + "revCount": 911303, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2505.810175%2Brev-b2a3852bd078e68dd2b3dfa8c00c67af1f0a7d20/01996a9c-be96-7a98-a978-e5a9ecbd877f/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2511.911303%2Brev-e07580dae39738e46609eaab8b154de2488133ce/019db12b-b7e1-7762-aadd-382b14c5b218/source.tar.gz" }, "original": { "type": "tarball", @@ -41,12 +41,12 @@ "nixpkgs": "nixpkgs_2" }, "locked": { - "lastModified": 1758508617, - "narHash": "sha256-kx2uELmVnAbiekj/YFfWR26OXqXedImkhe2ocnbumTA=", - "rev": "d2bac276ac7e669a1f09c48614538a37e3eb6d0f", - "revCount": 1924, + "lastModified": 1776827647, + "narHash": "sha256-sYixYhp5V8jCajO8TRorE4fzs7IkL4MZdfLTKgkPQBk=", + "rev": "40e6ccc06e1245a4837cbbd6bdda64e21cc67379", + "revCount": 2151, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/oxalica/rust-overlay/0.1.1924%2Brev-d2bac276ac7e669a1f09c48614538a37e3eb6d0f/01996f4a-36eb-7480-9a05-c8caee46ccc0/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/oxalica/rust-overlay/0.1.2151%2Brev-40e6ccc06e1245a4837cbbd6bdda64e21cc67379/019db32f-dc56-7292-a296-bbe69236be74/source.tar.gz" }, "original": { "type": "tarball", diff --git a/crates/numcodecs-wasm-builder/buildenv/rust-toolchain b/crates/numcodecs-wasm-builder/buildenv/rust-toolchain index 9c12d9413..5a7ad886f 100644 --- a/crates/numcodecs-wasm-builder/buildenv/rust-toolchain +++ b/crates/numcodecs-wasm-builder/buildenv/rust-toolchain @@ -1,5 +1,5 @@ [toolchain] -channel = "1.87" # MSRV +channel = "1.88" # MSRV components = [ "cargo", "rustfmt", "clippy", "rust-src" ] targets = [ "wasm32-wasip1" ] profile = "minimal" diff --git a/crates/numcodecs-wasm-guest/README.md b/crates/numcodecs-wasm-guest/README.md index 0bea44f3a..34337f695 100644 --- a/crates/numcodecs-wasm-guest/README.md +++ b/crates/numcodecs-wasm-guest/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-guest diff --git a/crates/numcodecs-wasm-guest/src/lib.rs b/crates/numcodecs-wasm-guest/src/lib.rs index 3462f2fa6..90880ff6c 100644 --- a/crates/numcodecs-wasm-guest/src/lib.rs +++ b/crates/numcodecs-wasm-guest/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-guest diff --git a/crates/numcodecs-wasm-host-reproducible/README.md b/crates/numcodecs-wasm-host-reproducible/README.md index e32257514..b8db18826 100644 --- a/crates/numcodecs-wasm-host-reproducible/README.md +++ b/crates/numcodecs-wasm-host-reproducible/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-host-reproducible diff --git a/crates/numcodecs-wasm-host-reproducible/src/lib.rs b/crates/numcodecs-wasm-host-reproducible/src/lib.rs index b3bec752e..3f721cead 100644 --- a/crates/numcodecs-wasm-host-reproducible/src/lib.rs +++ b/crates/numcodecs-wasm-host-reproducible/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-host-reproducible diff --git a/crates/numcodecs-wasm-host/README.md b/crates/numcodecs-wasm-host/README.md index e3d4c554c..df8e3333a 100644 --- a/crates/numcodecs-wasm-host/README.md +++ b/crates/numcodecs-wasm-host/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-host diff --git a/crates/numcodecs-wasm-host/src/lib.rs b/crates/numcodecs-wasm-host/src/lib.rs index fa57892c2..52b78eb82 100644 --- a/crates/numcodecs-wasm-host/src/lib.rs +++ b/crates/numcodecs-wasm-host/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-host diff --git a/crates/numcodecs-wasm-logging/README.md b/crates/numcodecs-wasm-logging/README.md index 1f82a67f1..41bd659ec 100644 --- a/crates/numcodecs-wasm-logging/README.md +++ b/crates/numcodecs-wasm-logging/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-logging diff --git a/crates/numcodecs-wasm-logging/src/lib.rs b/crates/numcodecs-wasm-logging/src/lib.rs index 9f073e03e..7972decd3 100644 --- a/crates/numcodecs-wasm-logging/src/lib.rs +++ b/crates/numcodecs-wasm-logging/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-logging diff --git a/crates/numcodecs/README.md b/crates/numcodecs/README.md index 1b8ebaa18..7e29e802c 100644 --- a/crates/numcodecs/README.md +++ b/crates/numcodecs/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs diff --git a/crates/numcodecs/src/lib.rs b/crates/numcodecs/src/lib.rs index b4db09d2b..d56fbe04d 100644 --- a/crates/numcodecs/src/lib.rs +++ b/crates/numcodecs/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs From 411c5ea873cbfda3df58eed92f09a51dcd7a2535 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 22 Apr 2026 07:10:18 +0300 Subject: [PATCH 46/55] Try removing bytemuck version pin --- codecs/fourier-network/Cargo.toml | 2 -- codecs/fourier-network/src/lib.rs | 3 --- 2 files changed, 5 deletions(-) diff --git a/codecs/fourier-network/Cargo.toml b/codecs/fourier-network/Cargo.toml index 8f2a6a775..5cce7e381 100644 --- a/codecs/fourier-network/Cargo.toml +++ b/codecs/fourier-network/Cargo.toml @@ -18,8 +18,6 @@ keywords = ["fourier", "network", "numcodecs", "compression", "encoding"] burn = { workspace = true, features = ["std", "autodiff", "ndarray"] } itertools = { workspace = true, features = ["use_alloc"] } log = { workspace = true } -# FIXME: bytemuck 1.24 fails to compile on 1.88 -bytemuck = { version = "=1.23.2", default-features = false } ndarray = { workspace = true, features = ["std"] } numcodecs = { workspace = true } num-traits = { workspace = true, features = ["std"] } diff --git a/codecs/fourier-network/src/lib.rs b/codecs/fourier-network/src/lib.rs index d2d1121f4..0ad9ff479 100644 --- a/codecs/fourier-network/src/lib.rs +++ b/codecs/fourier-network/src/lib.rs @@ -46,9 +46,6 @@ use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; -// FIXME: bytemuck 1.24 fails to compile on 1.88 -use ::bytemuck as _; - // FIXME: burn-common -> cubecl-common brings in wasm-bindgen // wasm-bindgen v0.2.115 has an unresolved import in wasm32-wasi use ::wasm_bindgen as _; From 807d7252ec3f69c4a4df04874a3685225a9f7dbc Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 22 Apr 2026 07:14:06 +0300 Subject: [PATCH 47/55] Try to remove wasm-bindgen pin --- codecs/fourier-network/Cargo.toml | 3 --- codecs/fourier-network/src/lib.rs | 4 ---- 2 files changed, 7 deletions(-) diff --git a/codecs/fourier-network/Cargo.toml b/codecs/fourier-network/Cargo.toml index 5cce7e381..6a888d17f 100644 --- a/codecs/fourier-network/Cargo.toml +++ b/codecs/fourier-network/Cargo.toml @@ -24,9 +24,6 @@ num-traits = { workspace = true, features = ["std"] } schemars = { workspace = true, features = ["derive", "preserve_order"] } serde = { workspace = true, features = ["std", "derive"] } thiserror = { workspace = true } -# FIXME: burn-common -> cubecl-common brings in wasm-bindgen -# wasm-bindgen v0.2.115 has an unresolved import in wasm32-wasi -wasm-bindgen = { version = "=0.2.114", default-features = false } [dev-dependencies] serde_json = { workspace = true, features = ["std"] } diff --git a/codecs/fourier-network/src/lib.rs b/codecs/fourier-network/src/lib.rs index 0ad9ff479..91d101399 100644 --- a/codecs/fourier-network/src/lib.rs +++ b/codecs/fourier-network/src/lib.rs @@ -46,10 +46,6 @@ use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; -// FIXME: burn-common -> cubecl-common brings in wasm-bindgen -// wasm-bindgen v0.2.115 has an unresolved import in wasm32-wasi -use ::wasm_bindgen as _; - #[cfg(test)] use ::serde_json as _; From ffbe07ec1e9c7d8768c49de7c2e1013171cfbc30 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 22 Apr 2026 09:29:47 +0300 Subject: [PATCH 48/55] Fix clippy --- Cargo.toml | 2 +- codecs/ebcc/tests/schema.rs | 1 + codecs/fourier-network/Cargo.toml | 2 ++ codecs/fourier-network/src/lib.rs | 3 +++ codecs/fourier-network/tests/config.rs | 4 ++-- codecs/fourier-network/tests/schema.rs | 2 +- codecs/jpeg2000/tests/config.rs | 2 +- codecs/jpeg2000/tests/schema.rs | 1 + codecs/pco/tests/config.rs | 2 +- codecs/pco/tests/schema.rs | 1 + codecs/pressio/tests/schema.rs | 5 +++-- codecs/qpet-sperr/src/lib.rs | 6 +++++- codecs/qpet-sperr/tests/config.rs | 2 +- codecs/qpet-sperr/tests/schema.rs | 1 + codecs/random-projection/tests/config.rs | 2 +- codecs/random-projection/tests/schema.rs | 1 + codecs/round/src/lib.rs | 8 +++++--- codecs/sperr/tests/config.rs | 6 +++++- codecs/sperr/tests/schema.rs | 1 + codecs/stochastic-rounding/src/lib.rs | 1 + codecs/swizzle-reshape/src/lib.rs | 5 ++++- codecs/sz3/src/lib.rs | 1 + codecs/sz3/tests/config.rs | 2 +- codecs/sz3/tests/schema.rs | 1 + codecs/tthresh/tests/config.rs | 2 +- codecs/tthresh/tests/schema.rs | 1 + codecs/zfp-classic/tests/config.rs | 2 +- codecs/zfp-classic/tests/schema.rs | 1 + codecs/zfp/tests/config.rs | 2 +- codecs/zfp/tests/schema.rs | 1 + crates/numcodecs-python/src/schema.rs | 14 +++++++------- crates/numcodecs-wasm-guest/src/lib.rs | 1 - .../src/tests.rs | 5 +++-- crates/numcodecs/src/codec.rs | 18 ++++++++---------- 34 files changed, 69 insertions(+), 40 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 512b5acba..7e5ef5b43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,7 +97,7 @@ fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "edf7511", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "018c80b", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/ebcc/tests/schema.rs b/codecs/ebcc/tests/schema.rs index a1b75551d..8d1d776ec 100644 --- a/codecs/ebcc/tests/schema.rs +++ b/codecs/ebcc/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("EBCC schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/fourier-network/Cargo.toml b/codecs/fourier-network/Cargo.toml index 6a888d17f..b4435445c 100644 --- a/codecs/fourier-network/Cargo.toml +++ b/codecs/fourier-network/Cargo.toml @@ -16,6 +16,8 @@ keywords = ["fourier", "network", "numcodecs", "compression", "encoding"] [dependencies] burn = { workspace = true, features = ["std", "autodiff", "ndarray"] } +# FIXME: bytemuck 1.24 fails to compile on 1.88 +bytemuck = { version = "=1.23.2", default-features = false } itertools = { workspace = true, features = ["use_alloc"] } log = { workspace = true } ndarray = { workspace = true, features = ["std"] } diff --git a/codecs/fourier-network/src/lib.rs b/codecs/fourier-network/src/lib.rs index 91d101399..fb87d32ae 100644 --- a/codecs/fourier-network/src/lib.rs +++ b/codecs/fourier-network/src/lib.rs @@ -46,6 +46,9 @@ use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; +// FIXME: bytemuck 1.24 fails to compile on 1.88 +use ::bytemuck as _; + #[cfg(test)] use ::serde_json as _; diff --git a/codecs/fourier-network/tests/config.rs b/codecs/fourier-network/tests/config.rs index 30d0c2d09..7c12d78f9 100644 --- a/codecs/fourier-network/tests/config.rs +++ b/codecs/fourier-network/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use numcodecs::StaticCodec; use numcodecs_fourier_network::FourierNetworkCodec; @@ -7,7 +7,7 @@ use serde_json::json; use ::{ burn as _, bytemuck as _, itertools as _, log as _, ndarray as _, num_traits as _, - schemars as _, simple_logger as _, thiserror as _, wasm_bindgen as _, + schemars as _, simple_logger as _, thiserror as _, }; #[test] diff --git a/codecs/fourier-network/tests/schema.rs b/codecs/fourier-network/tests/schema.rs index aec0f709a..4d9eeb338 100644 --- a/codecs/fourier-network/tests/schema.rs +++ b/codecs/fourier-network/tests/schema.rs @@ -6,7 +6,6 @@ use numcodecs_fourier_network::FourierNetworkCodec; use ::{ burn as _, bytemuck as _, itertools as _, log as _, ndarray as _, num_traits as _, schemars as _, serde as _, serde_json as _, simple_logger as _, thiserror as _, - wasm_bindgen as _, }; #[test] @@ -18,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("FourierNetwork schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/jpeg2000/tests/config.rs b/codecs/jpeg2000/tests/config.rs index 647901222..deb0d9e5b 100644 --- a/codecs/jpeg2000/tests/config.rs +++ b/codecs/jpeg2000/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use ::{ log as _, ndarray as _, num_traits as _, numcodecs_jpeg2000::Jpeg2000CompressionMode, diff --git a/codecs/jpeg2000/tests/schema.rs b/codecs/jpeg2000/tests/schema.rs index 393791d5b..474b627f7 100644 --- a/codecs/jpeg2000/tests/schema.rs +++ b/codecs/jpeg2000/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("Jpeg2000 schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/pco/tests/config.rs b/codecs/pco/tests/config.rs index 3f0d1d504..59c6a376a 100644 --- a/codecs/pco/tests/config.rs +++ b/codecs/pco/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use std::num::{NonZero, NonZeroUsize}; diff --git a/codecs/pco/tests/schema.rs b/codecs/pco/tests/schema.rs index e69a83500..2e1af9925 100644 --- a/codecs/pco/tests/schema.rs +++ b/codecs/pco/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("Pcodec schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/pressio/tests/schema.rs b/codecs/pressio/tests/schema.rs index ae8dde7ff..ec6b74557 100644 --- a/codecs/pressio/tests/schema.rs +++ b/codecs/pressio/tests/schema.rs @@ -1,8 +1,8 @@ #![expect(missing_docs)] use ::{ - libpressio as _, ndarray as _, schemars as _, serde as _, serde_json as _, serde_ndim as _, - thiserror as _, + fragile as _, libpressio as _, ndarray as _, schemars as _, serde as _, serde_json as _, + serde_ndim as _, thiserror as _, }; use numcodecs::{DynCodecType, StaticCodecType}; @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("Pressio schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/qpet-sperr/src/lib.rs b/codecs/qpet-sperr/src/lib.rs index 445bbe24a..f90764dfc 100644 --- a/codecs/qpet-sperr/src/lib.rs +++ b/codecs/qpet-sperr/src/lib.rs @@ -602,6 +602,7 @@ mod tests { #[test] fn all_modes() { + #[expect(clippy::single_element_loop)] for mode in [QpetSperrCompressionMode::SymbolicQuantityOfInterest { qoi: String::from("x^2"), qoi_block_size: default_qoi_block_size(), @@ -685,7 +686,10 @@ mod tests { let decoded = decompress(&encoded).unwrap(); assert_eq!(decoded.dtype(), AnyArrayDType::F64); - assert_eq!(decoded.len(), 64 * 64 * 1); + #[expect(clippy::identity_op)] + { + assert_eq!(decoded.len(), 64 * 64 * 1); + } assert_eq!(decoded.shape(), &[64, 64, 1]); } } diff --git a/codecs/qpet-sperr/tests/config.rs b/codecs/qpet-sperr/tests/config.rs index ac5e7a876..662a04179 100644 --- a/codecs/qpet-sperr/tests/config.rs +++ b/codecs/qpet-sperr/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use ::{ ndarray as _, num_traits as _, postcard as _, qpet_sperr as _, schemars as _, thiserror as _, diff --git a/codecs/qpet-sperr/tests/schema.rs b/codecs/qpet-sperr/tests/schema.rs index 6b95744e1..ff9746b8c 100644 --- a/codecs/qpet-sperr/tests/schema.rs +++ b/codecs/qpet-sperr/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("QPET-SPERR schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/random-projection/tests/config.rs b/codecs/random-projection/tests/config.rs index 728223ba8..102527d63 100644 --- a/codecs/random-projection/tests/config.rs +++ b/codecs/random-projection/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use numcodecs::StaticCodec; use numcodecs_random_projection::RandomProjectionCodec; diff --git a/codecs/random-projection/tests/schema.rs b/codecs/random-projection/tests/schema.rs index 8b42d1613..cf0d03370 100644 --- a/codecs/random-projection/tests/schema.rs +++ b/codecs/random-projection/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("RandomProjection schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/round/src/lib.rs b/codecs/round/src/lib.rs index aed61938f..91eb806f6 100644 --- a/codecs/round/src/lib.rs +++ b/codecs/round/src/lib.rs @@ -198,6 +198,7 @@ mod tests { } #[test] + #[expect(clippy::float_cmp)] fn round_minimal_precision() { let data = array![0.1, 1.0, 11.0, 21.0]; @@ -219,11 +220,11 @@ mod tests { 0.0, 0.1, 0.2, - 0.30000000000000004, + 0.300_000_000_000_000_04, 0.4, 0.5, - 0.6000000000000001, - 0.7000000000000001, + 0.600_000_000_000_000_1, + 0.700_000_000_000_000_1, 0.8, 0.9, 1.0 @@ -250,6 +251,7 @@ mod tests { let rounded = round(data.view(), NonNegative(1.0)); + #[expect(clippy::float_cmp)] for (d, r) in data.into_iter().zip(rounded) { assert!(d == r || d.to_bits() == r.to_bits()); } diff --git a/codecs/sperr/tests/config.rs b/codecs/sperr/tests/config.rs index aa471e553..e20ca5139 100644 --- a/codecs/sperr/tests/config.rs +++ b/codecs/sperr/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use ::{ndarray as _, num_traits as _, postcard as _, schemars as _, sperr as _, thiserror as _}; @@ -14,6 +14,7 @@ fn empty_config() { } #[test] +#[expect(clippy::float_cmp)] fn bpp_config() { let codec = SperrCodec::from_config( Deserialize::deserialize(json!({ @@ -30,6 +31,7 @@ fn bpp_config() { } #[test] +#[expect(clippy::float_cmp)] fn psnr_config() { let codec = SperrCodec::from_config( Deserialize::deserialize(json!({ @@ -46,6 +48,7 @@ fn psnr_config() { } #[test] +#[expect(clippy::float_cmp)] fn pwe_config() { let codec = SperrCodec::from_config( Deserialize::deserialize(json!({ @@ -62,6 +65,7 @@ fn pwe_config() { } #[test] +#[expect(clippy::float_cmp)] fn q_config() { let codec = SperrCodec::from_config( Deserialize::deserialize(json!({ diff --git a/codecs/sperr/tests/schema.rs b/codecs/sperr/tests/schema.rs index b25694ea2..1e2076f3b 100644 --- a/codecs/sperr/tests/schema.rs +++ b/codecs/sperr/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("Sperr schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/stochastic-rounding/src/lib.rs b/codecs/stochastic-rounding/src/lib.rs index 475c7e5ca..29844d212 100644 --- a/codecs/stochastic-rounding/src/lib.rs +++ b/codecs/stochastic-rounding/src/lib.rs @@ -333,6 +333,7 @@ mod tests { } #[test] + #[expect(clippy::float_cmp)] fn round_minimal_precision() { let data = array![0.1, 1.0, 11.0, 21.0]; diff --git a/codecs/swizzle-reshape/src/lib.rs b/codecs/swizzle-reshape/src/lib.rs index 850d4c4d9..ed8d3c3a4 100644 --- a/codecs/swizzle-reshape/src/lib.rs +++ b/codecs/swizzle-reshape/src/lib.rs @@ -779,7 +779,10 @@ mod tests { Axis::Index(2), Axis::Index(1), ])], - &[1 * 3 * 1 * 721 * 1440], + #[expect(clippy::identity_op)] + { + &[1 * 3 * 1 * 721 * 1440] + }, ); } diff --git a/codecs/sz3/src/lib.rs b/codecs/sz3/src/lib.rs index b783e5dc7..e7c76ce0a 100644 --- a/codecs/sz3/src/lib.rs +++ b/codecs/sz3/src/lib.rs @@ -859,6 +859,7 @@ mod tests { } #[test] + #[expect(clippy::redundant_closure)] // FIXME fn all_dtypes() -> Result<(), Sz3CodecError> { fn compress_decompress( iter: impl Clone + IntoIterator, diff --git a/codecs/sz3/tests/config.rs b/codecs/sz3/tests/config.rs index aa5bda0f2..e34d85ff4 100644 --- a/codecs/sz3/tests/config.rs +++ b/codecs/sz3/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use ::{ ndarray as _, num_traits as _, postcard as _, schemars as _, sz3 as _, thiserror as _, diff --git a/codecs/sz3/tests/schema.rs b/codecs/sz3/tests/schema.rs index 9e40080e0..0aab3214a 100644 --- a/codecs/sz3/tests/schema.rs +++ b/codecs/sz3/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("Sz3 schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/tthresh/tests/config.rs b/codecs/tthresh/tests/config.rs index 382fc43ab..3dbeb7fbf 100644 --- a/codecs/tthresh/tests/config.rs +++ b/codecs/tthresh/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use ::{ndarray as _, num_traits as _, schemars as _, thiserror as _, tthresh as _}; diff --git a/codecs/tthresh/tests/schema.rs b/codecs/tthresh/tests/schema.rs index 20327bf24..5a5d9167a 100644 --- a/codecs/tthresh/tests/schema.rs +++ b/codecs/tthresh/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("Tthresh schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/zfp-classic/tests/config.rs b/codecs/zfp-classic/tests/config.rs index e21ddc33e..944744952 100644 --- a/codecs/zfp-classic/tests/config.rs +++ b/codecs/zfp-classic/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use ::{ndarray as _, postcard as _, schemars as _, thiserror as _, zfp_sys as _}; diff --git a/codecs/zfp-classic/tests/schema.rs b/codecs/zfp-classic/tests/schema.rs index 053227517..d6240fd48 100644 --- a/codecs/zfp-classic/tests/schema.rs +++ b/codecs/zfp-classic/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("ZfpClassic schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/zfp/tests/config.rs b/codecs/zfp/tests/config.rs index dee661f09..ffe89d915 100644 --- a/codecs/zfp/tests/config.rs +++ b/codecs/zfp/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use ::{ndarray as _, postcard as _, schemars as _, thiserror as _, zfp_sys as _}; diff --git a/codecs/zfp/tests/schema.rs b/codecs/zfp/tests/schema.rs index 9d8bf4d7b..be941cab9 100644 --- a/codecs/zfp/tests/schema.rs +++ b/codecs/zfp/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("Zfp schema has changed\n===\n{schema}\n==="); } diff --git a/crates/numcodecs-python/src/schema.rs b/crates/numcodecs-python/src/schema.rs index e43ad0261..ea728e8d3 100644 --- a/crates/numcodecs-python/src/schema.rs +++ b/crates/numcodecs-python/src/schema.rs @@ -108,13 +108,13 @@ pub fn schema_from_codec_class( schema.insert(String::from("additionalProperties"), Value::Bool(true)); } - if let Ok(doc) = class.getattr(intern!(py, "__doc__")) { - if !doc.is_none() { - let doc: String = doc - .extract() - .map_err(|err| SchemaError::InvalidClassDocs { source: err })?; - schema.insert(String::from("description"), Value::String(doc)); - } + if let Ok(doc) = class.getattr(intern!(py, "__doc__")) + && !doc.is_none() + { + let doc: String = doc + .extract() + .map_err(|err| SchemaError::InvalidClassDocs { source: err })?; + schema.insert(String::from("description"), Value::String(doc)); } let name = class diff --git a/crates/numcodecs-wasm-guest/src/lib.rs b/crates/numcodecs-wasm-guest/src/lib.rs index 90880ff6c..cd4601ed6 100644 --- a/crates/numcodecs-wasm-guest/src/lib.rs +++ b/crates/numcodecs-wasm-guest/src/lib.rs @@ -44,7 +44,6 @@ use crate::{ }; #[doc(hidden)] -#[expect(clippy::same_length_and_capacity)] pub mod bindings { wit_bindgen::generate!({ world: "numcodecs:abc/exports@0.1.1", diff --git a/crates/numcodecs-wasm-host-reproducible/src/tests.rs b/crates/numcodecs-wasm-host-reproducible/src/tests.rs index d22978864..260eab5d5 100644 --- a/crates/numcodecs-wasm-host-reproducible/src/tests.rs +++ b/crates/numcodecs-wasm-host-reproducible/src/tests.rs @@ -13,6 +13,7 @@ const MEMORY_GUARD_SIZE: u32 = WASM_PAGE_SIZE * 16 * 64 /* 64MiB */; const MEMORY_RESERVATION_FOR_GROWTH: u32 = WASM_PAGE_SIZE * 16 * 64 /* 64MiB */; #[test] +#[expect(clippy::unwrap_used, clippy::panic)] fn codec_roundtrip() { // keep in sync with numcodecs-wasm let mut config = wasmtime::Config::new(); @@ -71,7 +72,7 @@ fn codec_roundtrip() { let data = Array::random((256, 256), Normal::new(0.0, 1.0).unwrap()); - let encoded = match codec.encode(numcodecs::AnyArray::F64(data.clone().into_dyn()).into_cow()) { + let encoded = match codec.encode(numcodecs::AnyArray::F64(data.into_dyn()).into_cow()) { Ok(encoded) => encoded, Err(err) => panic!( "ReproducibleWasmCodec::encode:\n===\n{err}\n===\n{err:?}\n===\n{err:#}\n===\n{err:#?}\n===\n" @@ -85,7 +86,7 @@ fn codec_roundtrip() { Err(err) => panic!( "ReproducibleWasmCodec::decode_into:\n===\n{err}\n===\n{err:?}\n===\n{err:#}\n===\n{err:#?}\n===\n" ), - }; + } let decoded = match codec.decode(encoded.into_cow()) { Ok(decoded) => decoded, diff --git a/crates/numcodecs/src/codec.rs b/crates/numcodecs/src/codec.rs index ee87e241d..0e144615c 100644 --- a/crates/numcodecs/src/codec.rs +++ b/crates/numcodecs/src/codec.rs @@ -283,16 +283,14 @@ pub fn codec_from_config_with_id<'de, T: DynCodecType, D: Deserializer<'de>>( ) -> Result { let mut config = Value::deserialize(config)?; - if let Some(config) = config.as_object_mut() { - if let Some(id) = config.remove("id") { - let codec_id = ty.codec_id(); - - if !matches!(id, Value::String(ref id) if id == codec_id) { - return Err(serde::de::Error::custom(format!( - "expected codec id {codec_id:?} but found {id}" - ))); - } - } + if let Some(config) = config.as_object_mut() + && let Some(id) = config.remove("id") + && let codec_id = ty.codec_id() + && !matches!(id, Value::String(ref id) if id == codec_id) + { + return Err(serde::de::Error::custom(format!( + "expected codec id {codec_id:?} but found {id}" + ))); } ty.codec_from_config(config) From 0a89029586b4086cc97c17787e15fe3d981fed4c Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 22 Apr 2026 09:41:14 +0300 Subject: [PATCH 49/55] add serde(deny_unknown_fields) for pressio config --- codecs/fourier-network/Cargo.toml | 3 +++ codecs/fourier-network/src/lib.rs | 4 ++++ codecs/fourier-network/tests/config.rs | 2 +- codecs/fourier-network/tests/schema.rs | 1 + codecs/pressio/src/lib.rs | 2 ++ codecs/pressio/tests/schema.json | 6 +++--- 6 files changed, 14 insertions(+), 4 deletions(-) diff --git a/codecs/fourier-network/Cargo.toml b/codecs/fourier-network/Cargo.toml index b4435445c..2d8cc71f7 100644 --- a/codecs/fourier-network/Cargo.toml +++ b/codecs/fourier-network/Cargo.toml @@ -26,6 +26,9 @@ num-traits = { workspace = true, features = ["std"] } schemars = { workspace = true, features = ["derive", "preserve_order"] } serde = { workspace = true, features = ["std", "derive"] } thiserror = { workspace = true } +# FIXME: burn-common -> cubecl-common brings in wasm-bindgen +# wasm-bindgen v0.2.115 has an unresolved import in wasm32-wasi +wasm-bindgen = { version = "=0.2.114", default-features = false } [dev-dependencies] serde_json = { workspace = true, features = ["std"] } diff --git a/codecs/fourier-network/src/lib.rs b/codecs/fourier-network/src/lib.rs index fb87d32ae..d2d1121f4 100644 --- a/codecs/fourier-network/src/lib.rs +++ b/codecs/fourier-network/src/lib.rs @@ -49,6 +49,10 @@ use thiserror::Error; // FIXME: bytemuck 1.24 fails to compile on 1.88 use ::bytemuck as _; +// FIXME: burn-common -> cubecl-common brings in wasm-bindgen +// wasm-bindgen v0.2.115 has an unresolved import in wasm32-wasi +use ::wasm_bindgen as _; + #[cfg(test)] use ::serde_json as _; diff --git a/codecs/fourier-network/tests/config.rs b/codecs/fourier-network/tests/config.rs index 7c12d78f9..ab78ad450 100644 --- a/codecs/fourier-network/tests/config.rs +++ b/codecs/fourier-network/tests/config.rs @@ -7,7 +7,7 @@ use serde_json::json; use ::{ burn as _, bytemuck as _, itertools as _, log as _, ndarray as _, num_traits as _, - schemars as _, simple_logger as _, thiserror as _, + schemars as _, simple_logger as _, thiserror as _, wasm_bindgen as _, }; #[test] diff --git a/codecs/fourier-network/tests/schema.rs b/codecs/fourier-network/tests/schema.rs index 4d9eeb338..7ce541e06 100644 --- a/codecs/fourier-network/tests/schema.rs +++ b/codecs/fourier-network/tests/schema.rs @@ -6,6 +6,7 @@ use numcodecs_fourier_network::FourierNetworkCodec; use ::{ burn as _, bytemuck as _, itertools as _, log as _, ndarray as _, num_traits as _, schemars as _, serde as _, serde_json as _, simple_logger as _, thiserror as _, + wasm_bindgen as _, }; #[test] diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 2da4148a3..ea1e281c2 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -485,6 +485,7 @@ impl JsonSchema for PressioCompressor { #[derive(Debug, Deserialize, JsonSchema)] #[serde(rename = "PressioCompressor")] +#[serde(deny_unknown_fields)] struct PressioCompressorOwnedFormat { /// The id of the compressor compressor_id: String, @@ -504,6 +505,7 @@ struct PressioCompressorOwnedFormat { #[derive(Debug, Serialize)] #[serde(rename = "PressioCompressor")] +#[serde(deny_unknown_fields)] struct PressioCompressorBorrowedFormat<'a> { /// The id of the compressor compressor_id: &'a str, diff --git a/codecs/pressio/tests/schema.json b/codecs/pressio/tests/schema.json index 0cec2dfe5..7cb424a94 100644 --- a/codecs/pressio/tests/schema.json +++ b/codecs/pressio/tests/schema.json @@ -1,6 +1,9 @@ { "type": "object", "additionalProperties": false, + "required": [ + "compressor_id" + ], "properties": { "compressor_id": { "type": "string", @@ -852,9 +855,6 @@ "default": "1.0.0" } }, - "required": [ - "compressor_id" - ], "description": "Pressio codec which applies the identity function, i.e. passes through the\ninput unchanged during encoding and decoding.", "title": "PressioCodec", "$schema": "https://json-schema.org/draft/2020-12/schema", From 29780c83d83551b2876a4f46adf68b8ad074ab2f Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 22 Apr 2026 10:03:40 +0300 Subject: [PATCH 50/55] Fix clippy --- crates/numcodecs-wasm-guest/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/numcodecs-wasm-guest/src/lib.rs b/crates/numcodecs-wasm-guest/src/lib.rs index cd4601ed6..90880ff6c 100644 --- a/crates/numcodecs-wasm-guest/src/lib.rs +++ b/crates/numcodecs-wasm-guest/src/lib.rs @@ -44,6 +44,7 @@ use crate::{ }; #[doc(hidden)] +#[expect(clippy::same_length_and_capacity)] pub mod bindings { wit_bindgen::generate!({ world: "numcodecs:abc/exports@0.1.1", From 646e05c86e1d9f2e03d0c90298aad2fb4d7fa433 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 23 Apr 2026 10:11:25 +0300 Subject: [PATCH 51/55] Improve codec schema inlining --- Cargo.toml | 4 +- codecs/ebcc/src/lib.rs | 4 + codecs/fourier-network/src/lib.rs | 4 + codecs/lc/src/lib.rs | 6 + codecs/lc/tests/schema.json | 192 +-- codecs/pco/src/lib.rs | 2 + codecs/pressio/tests/schema.json | 1220 +++----------------- codecs/qpet-sperr/src/lib.rs | 4 + codecs/random-projection/src/lib.rs | 10 +- codecs/random-projection/tests/schema.json | 19 +- codecs/sperr/src/lib.rs | 4 + codecs/swizzle-reshape/Cargo.toml | 3 + codecs/swizzle-reshape/src/lib.rs | 9 + codecs/swizzle-reshape/tests/schema.json | 58 + codecs/swizzle-reshape/tests/schema.rs | 21 + codecs/sz3/src/lib.rs | 1 + codecs/tthresh/src/lib.rs | 4 + codecs/zfp-classic/src/lib.rs | 1 + codecs/zfp/src/lib.rs | 1 + crates/numcodecs-wasm-guest/Cargo.toml | 2 +- crates/numcodecs-wasm-guest/src/lib.rs | 8 +- crates/numcodecs/Cargo.toml | 2 +- crates/numcodecs/src/codec.rs | 16 +- 23 files changed, 354 insertions(+), 1241 deletions(-) create mode 100644 codecs/swizzle-reshape/tests/schema.json create mode 100644 codecs/swizzle-reshape/tests/schema.rs diff --git a/Cargo.toml b/Cargo.toml index 7e5ef5b43..b625fa8b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,10 +48,10 @@ rust-version = "1.88" [workspace.dependencies] # workspace-internal numcodecs crates -numcodecs = { version = "0.3.1", path = "crates/numcodecs", default-features = false } +numcodecs = { version = "0.3.2", path = "crates/numcodecs", default-features = false } numcodecs-python = { version = "0.7.1", path = "crates/numcodecs-python", default-features = false } numcodecs-wasm-builder = { version = "0.2", path = "crates/numcodecs-wasm-builder", default-features = false } -numcodecs-wasm-guest = { version = "0.3", path = "crates/numcodecs-wasm-guest", default-features = false } +numcodecs-wasm-guest = { version = "0.3.1", path = "crates/numcodecs-wasm-guest", default-features = false } numcodecs-wasm-host = { version = "0.2", path = "crates/numcodecs-wasm-host", default-features = false } numcodecs-wasm-host-reproducible = { version = "0.2.2", path = "crates/numcodecs-wasm-host-reproducible", default-features = false } numcodecs-wasm-logging = { version = "0.2", path = "crates/numcodecs-wasm-logging", default-features = false } diff --git a/codecs/ebcc/src/lib.rs b/codecs/ebcc/src/lib.rs index fb35b5704..57a574092 100644 --- a/codecs/ebcc/src/lib.rs +++ b/codecs/ebcc/src/lib.rs @@ -263,6 +263,10 @@ impl<'de> Deserialize<'de> for Positive { } impl JsonSchema for Positive { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("PositiveF32") } diff --git a/codecs/fourier-network/src/lib.rs b/codecs/fourier-network/src/lib.rs index d2d1121f4..b6489a18b 100644 --- a/codecs/fourier-network/src/lib.rs +++ b/codecs/fourier-network/src/lib.rs @@ -222,6 +222,10 @@ impl<'de> Deserialize<'de> for Positive { } impl JsonSchema for Positive { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("PositiveF64") } diff --git a/codecs/lc/src/lib.rs b/codecs/lc/src/lib.rs index 10e96fed4..71b1f6b01 100644 --- a/codecs/lc/src/lib.rs +++ b/codecs/lc/src/lib.rs @@ -75,6 +75,7 @@ fn deserialize_components<'de, D: Deserializer<'de>>( #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] #[serde(tag = "id")] +#[schemars(inline)] /// LC preprocessor pub enum LcPreprocessor { #[serde(rename = "NUL")] @@ -118,6 +119,7 @@ impl LcPreprocessor { #[derive( Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema, )] +#[schemars(inline)] /// LC error bound kind pub enum LcErrorKind { /// pointwise absolute error bound @@ -145,6 +147,7 @@ impl LcErrorKind { #[derive( Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema, )] +#[schemars(inline)] /// LC quantisation decorrelation mode pub enum LcDecorrelation { #[serde(rename = "0")] @@ -166,6 +169,7 @@ impl LcDecorrelation { #[derive( Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema, )] +#[schemars(inline)] /// LC Lorenzo preprocessor dtype pub enum LcLorenzoDtype { #[serde(rename = "i32")] @@ -184,6 +188,7 @@ impl LcLorenzoDtype { #[derive( Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema, )] +#[schemars(inline)] /// LC quantization dtype pub enum LcQuantizeDType { #[serde(rename = "f32")] @@ -207,6 +212,7 @@ impl LcQuantizeDType { )] #[serde(deny_unknown_fields)] #[serde(tag = "id")] +#[schemars(inline)] /// LC component pub enum LcComponent { #[serde(rename = "NUL")] diff --git a/codecs/lc/tests/schema.json b/codecs/lc/tests/schema.json index 3de092cbf..2c29e9cf0 100644 --- a/codecs/lc/tests/schema.json +++ b/codecs/lc/tests/schema.json @@ -132,14 +132,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -156,14 +149,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -180,12 +166,7 @@ "type": "object", "properties": { "size": { - "description": "LC component float element size, in bytes", - "type": "integer", - "enum": [ - 4, - 8 - ] + "$ref": "#/$defs/LcFloatSize" }, "id": { "type": "string", @@ -202,12 +183,7 @@ "type": "object", "properties": { "size": { - "description": "LC component float element size, in bytes", - "type": "integer", - "enum": [ - 4, - 8 - ] + "$ref": "#/$defs/LcFloatSize" }, "id": { "type": "string", @@ -224,14 +200,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -248,24 +217,7 @@ "type": "object", "properties": { "size": { - "description": "LC tuple component element size, in tuple length _ bytes", - "type": "string", - "enum": [ - "2_1", - "3_1", - "4_1", - "6_1", - "8_1", - "12_1", - "2_2", - "3_2", - "4_2", - "6_2", - "2_4", - "6_4", - "3_8", - "6_8" - ] + "$ref": "#/$defs/LcTupleSize" }, "id": { "type": "string", @@ -282,14 +234,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -306,14 +251,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -330,14 +268,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -354,14 +285,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -378,14 +302,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -402,14 +319,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -426,14 +336,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -450,14 +353,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -474,14 +370,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -498,14 +387,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -536,5 +418,45 @@ ], "description": "Codec providing compression using LC", "title": "LcCodec", - "$schema": "https://json-schema.org/draft/2020-12/schema" + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "LcElemSize": { + "type": "integer", + "enum": [ + 1, + 2, + 4, + 8 + ], + "description": "LC component element size, in bytes" + }, + "LcFloatSize": { + "type": "integer", + "enum": [ + 4, + 8 + ], + "description": "LC component float element size, in bytes" + }, + "LcTupleSize": { + "type": "string", + "enum": [ + "2_1", + "3_1", + "4_1", + "6_1", + "8_1", + "12_1", + "2_2", + "3_2", + "4_2", + "6_2", + "2_4", + "6_4", + "3_8", + "6_8" + ], + "description": "LC tuple component element size, in tuple length _ bytes" + } + } } \ No newline at end of file diff --git a/codecs/pco/src/lib.rs b/codecs/pco/src/lib.rs index 35d7d98ae..59541e184 100644 --- a/codecs/pco/src/lib.rs +++ b/codecs/pco/src/lib.rs @@ -61,6 +61,7 @@ pub struct Pcodec { Copy, Clone, Debug, Default, PartialEq, Eq, Serialize_repr, Deserialize_repr, JsonSchema_repr, )] #[repr(u8)] +#[schemars(inline)] /// Pco compression level. /// /// The level ranges from 0 to 12 inclusive (default: 8): @@ -157,6 +158,7 @@ pub enum PcoDeltaSpec { #[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize_repr, Deserialize_repr, JsonSchema_repr)] #[repr(u8)] +#[schemars(inline)] /// Pco delta encoding order. /// /// The order ranges from 0 to 7 inclusive. diff --git a/codecs/pressio/tests/schema.json b/codecs/pressio/tests/schema.json index 7cb424a94..fc99ab36d 100644 --- a/codecs/pressio/tests/schema.json +++ b/codecs/pressio/tests/schema.json @@ -12,276 +12,7 @@ "early_config": { "type": "object", "additionalProperties": { - "anyOf": [ - { - "type": "null" - }, - { - "type": "boolean" - }, - { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - }, - { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - }, - { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - }, - { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - }, - { - "type": "integer", - "format": "uint32", - "minimum": 0 - }, - { - "type": "integer", - "format": "int32" - }, - { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - { - "type": "integer", - "format": "int64" - }, - { - "type": "number", - "format": "float" - }, - { - "type": "number", - "format": "double" - }, - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "boolean" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/boolNdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint32", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint64", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int32" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int64" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "float" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "double" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f64NdArray" - } - } - ] - }, - { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/PressioOption" - } - } - ], - "description": "Pressio option value" + "$ref": "#/$defs/PressioOption" }, "description": "Configuration for the structure of the compressor", "default": {} @@ -289,276 +20,7 @@ "compressor_config": { "type": "object", "additionalProperties": { - "anyOf": [ - { - "type": "null" - }, - { - "type": "boolean" - }, - { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - }, - { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - }, - { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - }, - { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - }, - { - "type": "integer", - "format": "uint32", - "minimum": 0 - }, - { - "type": "integer", - "format": "int32" - }, - { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - { - "type": "integer", - "format": "int64" - }, - { - "type": "number", - "format": "float" - }, - { - "type": "number", - "format": "double" - }, - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "boolean" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/boolNdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint32", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint64", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int32" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int64" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "float" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "double" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f64NdArray" - } - } - ] - }, - { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/PressioOption" - } - } - ], - "description": "Pressio option value" + "$ref": "#/$defs/PressioOption" }, "description": "Configuration for the compressor", "default": {} @@ -566,276 +28,7 @@ "metric_results": { "type": "object", "additionalProperties": { - "anyOf": [ - { - "type": "null" - }, - { - "type": "boolean" - }, - { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - }, - { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - }, - { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - }, - { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - }, - { - "type": "integer", - "format": "uint32", - "minimum": 0 - }, - { - "type": "integer", - "format": "int32" - }, - { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - { - "type": "integer", - "format": "int64" - }, - { - "type": "number", - "format": "float" - }, - { - "type": "number", - "format": "double" - }, - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "boolean" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/boolNdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint32", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint64", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int32" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int64" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "float" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "double" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f64NdArray" - } - } - ] - }, - { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/PressioOption" - } - } - ], - "description": "Pressio option value" + "$ref": "#/$defs/PressioOption" }, "description": "Results of the compressor metrics (output-only)", "default": {} @@ -859,6 +52,141 @@ "title": "PressioCodec", "$schema": "https://json-schema.org/draft/2020-12/schema", "$defs": { + "PressioOption": { + "anyOf": [ + { + "type": "null", + "title": "None" + }, + { + "type": "boolean", + "title": "Bool" + }, + { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "title": "U8" + }, + { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127, + "title": "I8" + }, + { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535, + "title": "U16" + }, + { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767, + "title": "I16" + }, + { + "type": "integer", + "format": "uint32", + "minimum": 0, + "title": "U32" + }, + { + "type": "integer", + "format": "int32", + "title": "I32" + }, + { + "type": "integer", + "format": "uint64", + "minimum": 0, + "title": "U64" + }, + { + "type": "integer", + "format": "int64", + "title": "I64" + }, + { + "type": "number", + "format": "float", + "title": "F32" + }, + { + "type": "number", + "format": "double", + "title": "F64" + }, + { + "type": "string", + "title": "String" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "title": "VecString" + }, + { + "$ref": "#/$defs/boolNdArray", + "title": "DataBool" + }, + { + "$ref": "#/$defs/u8NdArray", + "title": "DataU8" + }, + { + "$ref": "#/$defs/u16NdArray", + "title": "DataU16" + }, + { + "$ref": "#/$defs/u32NdArray", + "title": "DataU32" + }, + { + "$ref": "#/$defs/u64NdArray", + "title": "DataU64" + }, + { + "$ref": "#/$defs/i8NdArray", + "title": "DataI8" + }, + { + "$ref": "#/$defs/i16NdArray", + "title": "DataI16" + }, + { + "$ref": "#/$defs/i32NdArray", + "title": "DataI32" + }, + { + "$ref": "#/$defs/i64NdArray", + "title": "DataI64" + }, + { + "$ref": "#/$defs/f32NdArray", + "title": "DataF32" + }, + { + "$ref": "#/$defs/f64NdArray", + "title": "DataF64" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/PressioOption" + }, + "title": "Nested" + } + ], + "description": "Pressio option value" + }, "boolNdArray": { "anyOf": [ { @@ -1054,278 +382,6 @@ } } ] - }, - "PressioOption": { - "anyOf": [ - { - "type": "null" - }, - { - "type": "boolean" - }, - { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - }, - { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - }, - { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - }, - { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - }, - { - "type": "integer", - "format": "uint32", - "minimum": 0 - }, - { - "type": "integer", - "format": "int32" - }, - { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - { - "type": "integer", - "format": "int64" - }, - { - "type": "number", - "format": "float" - }, - { - "type": "number", - "format": "double" - }, - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "boolean" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/boolNdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint32", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint64", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int32" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int64" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "float" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "double" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f64NdArray" - } - } - ] - }, - { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/PressioOption" - } - } - ], - "description": "Pressio option value" } } } \ No newline at end of file diff --git a/codecs/qpet-sperr/src/lib.rs b/codecs/qpet-sperr/src/lib.rs index f90764dfc..7384bbf0c 100644 --- a/codecs/qpet-sperr/src/lib.rs +++ b/codecs/qpet-sperr/src/lib.rs @@ -482,6 +482,10 @@ impl<'de> Deserialize<'de> for Positive { } impl JsonSchema for Positive { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("PositiveF64") } diff --git a/codecs/random-projection/src/lib.rs b/codecs/random-projection/src/lib.rs index 4510758bc..9761386d1 100644 --- a/codecs/random-projection/src/lib.rs +++ b/codecs/random-projection/src/lib.rs @@ -48,7 +48,7 @@ use ::serde_json as _; /// /// This codec only supports finite floating point data. #[derive(Clone, Serialize, Deserialize, JsonSchema)] -// FIXME: #[serde(deny_unknown_fields)] +#[schemars(deny_unknown_fields)] // serde cannot deny unknown fields because of the flatten pub struct RandomProjectionCodec { /// Seed for generating the random projection matrix pub seed: u64, @@ -65,7 +65,7 @@ pub struct RandomProjectionCodec { /// Method with which the reduced dimensionality `$K$` is selected #[derive(Clone, Serialize, Deserialize, JsonSchema)] -// FIXME: #[serde(deny_unknown_fields)] +#[schemars(deny_unknown_fields)] // serde cannot deny unknown fields because of the flatten #[serde(tag = "reduction", rename_all = "kebab-case")] pub enum RandomProjectionReduction { /// The reduced dimensionality `$K$` is derived from `epsilon`, as defined @@ -84,7 +84,7 @@ pub enum RandomProjectionReduction { /// Projection kind that is used to generate the random projection matrix #[derive(Clone, Serialize, Deserialize, JsonSchema)] -// FIXME: #[serde(deny_unknown_fields)] +#[schemars(deny_unknown_fields)] // serde cannot deny unknown fields because of the flatten #[serde(tag = "projection", rename_all = "kebab-case")] pub enum RandomProjectionKind { /// The random projection matrix is dense and its components are sampled @@ -799,6 +799,10 @@ impl<'de> Deserialize<'de> for OpenClosedUnit { } impl JsonSchema for OpenClosedUnit { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("OpenClosedUnitF64") } diff --git a/codecs/random-projection/tests/schema.json b/codecs/random-projection/tests/schema.json index 9f48b20f6..5f6aa5031 100644 --- a/codecs/random-projection/tests/schema.json +++ b/codecs/random-projection/tests/schema.json @@ -1,5 +1,8 @@ { "type": "object", + "required": [ + "seed" + ], "properties": { "seed": { "type": "integer", @@ -14,15 +17,14 @@ "default": "0.1.0" } }, - "required": [ - "seed" - ], + "unevaluatedProperties": false, "description": "Codec that uses random projections to reduce the dimensionality of high-\ndimensional data to compress it.\n\nA two-dimensional array of shape `$N \\times D$` is encoded as n array of\nshape `$N \\times K$`, where `$K$` is either set explicitly or chosen using\nthe the Johnson-Lindenstrauss lemma. For `$K$` to be smaller than `$D$`,\n`$D$` must be quite large. Therefore, this codec should only applied on\nlarge datasets as it otherwise significantly inflates the data size instead\nof reducing it.\n\nChoosing a lower distortion rate `epsilon` will improve the quality of the\nlossy compression, i.e. reduce the compression error, at the cost of\nincreasing `$K$`.\n\nThis codec only supports finite floating point data.", "allOf": [ { "oneOf": [ { "type": "object", + "description": "The reduced dimensionality `$K$` is derived from `epsilon`, as defined\nby the Johnson-Lindenstrauss lemma.", "properties": { "epsilon": { "type": "number", @@ -38,11 +40,11 @@ "required": [ "reduction", "epsilon" - ], - "description": "The reduced dimensionality `$K$` is derived from `epsilon`, as defined\nby the Johnson-Lindenstrauss lemma." + ] }, { "type": "object", + "description": "The reduced dimensionality `$K$`, to which the data is projected, is\ngiven explicitly.", "properties": { "k": { "type": "integer", @@ -58,8 +60,7 @@ "required": [ "reduction", "k" - ], - "description": "The reduced dimensionality `$K$`, to which the data is projected, is\ngiven explicitly." + ] } ] }, @@ -80,6 +81,7 @@ }, { "type": "object", + "description": "The random projection matrix is sparse where only `density`% of entries\nare non-zero.\n\nThe matrix's components are sampled from\n\n- `$-\\sqrt{\\frac{1}{k \\cdot density}}$` with probability\n `$0.5 \\cdot density$`\n- `$0$` with probability `$1 - density$`\n- `$+\\sqrt{\\frac{1}{k \\cdot density}}$` with probability\n `$0.5 \\cdot density$`", "properties": { "density": { "type": [ @@ -97,8 +99,7 @@ }, "required": [ "projection" - ], - "description": "The random projection matrix is sparse where only `density`% of entries\nare non-zero.\n\nThe matrix's components are sampled from\n\n- `$-\\sqrt{\\frac{1}{k \\cdot density}}$` with probability\n `$0.5 \\cdot density$`\n- `$0$` with probability `$1 - density$`\n- `$+\\sqrt{\\frac{1}{k \\cdot density}}$` with probability\n `$0.5 \\cdot density$`" + ] } ] } diff --git a/codecs/sperr/src/lib.rs b/codecs/sperr/src/lib.rs index 8195807c5..3ced82e19 100644 --- a/codecs/sperr/src/lib.rs +++ b/codecs/sperr/src/lib.rs @@ -451,6 +451,10 @@ impl<'de> Deserialize<'de> for Positive { } impl JsonSchema for Positive { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("PositiveF64") } diff --git a/codecs/swizzle-reshape/Cargo.toml b/codecs/swizzle-reshape/Cargo.toml index 469671be4..3b07fb1cb 100644 --- a/codecs/swizzle-reshape/Cargo.toml +++ b/codecs/swizzle-reshape/Cargo.toml @@ -19,5 +19,8 @@ schemars = { workspace = true, features = ["derive", "preserve_order"] } serde = { workspace = true, features = ["std", "derive"] } thiserror = { workspace = true } +[dev-dependencies] +serde_json = { workspace = true, features = ["std"] } + [lints] workspace = true diff --git a/codecs/swizzle-reshape/src/lib.rs b/codecs/swizzle-reshape/src/lib.rs index ed8d3c3a4..5d8534644 100644 --- a/codecs/swizzle-reshape/src/lib.rs +++ b/codecs/swizzle-reshape/src/lib.rs @@ -36,6 +36,9 @@ use serde::{ }; use thiserror::Error; +#[cfg(test)] +use ::serde_json as _; + #[derive(Clone, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] /// Codec to swizzle/swap the axes of an array and reshape it. @@ -71,6 +74,7 @@ pub struct SwizzleReshapeCodec { #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(untagged)] #[serde(deny_unknown_fields)] +#[schemars(inline)] /// An axis group, potentially from a merged combination of multiple input axes pub enum AxisGroup { /// A merged combination of zero, one, or multiple input axes @@ -82,6 +86,7 @@ pub enum AxisGroup { #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(untagged)] #[serde(deny_unknown_fields)] +#[schemars(inline)] /// An axis or all remaining axes pub enum Axis { /// A single axis, as determined by its index @@ -534,6 +539,10 @@ impl<'de> Deserialize<'de> for Rest { } impl JsonSchema for Rest { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("Rest") } diff --git a/codecs/swizzle-reshape/tests/schema.json b/codecs/swizzle-reshape/tests/schema.json new file mode 100644 index 000000000..3bdb0eaa7 --- /dev/null +++ b/codecs/swizzle-reshape/tests/schema.json @@ -0,0 +1,58 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "axes": { + "type": "array", + "items": { + "description": "An axis group, potentially from a merged combination of multiple input axes", + "anyOf": [ + { + "title": "Group", + "description": "A merged combination of zero, one, or multiple input axes", + "type": "array", + "items": { + "description": "An axis or all remaining axes", + "anyOf": [ + { + "title": "Index", + "description": "A single axis, as determined by its index", + "type": "integer", + "format": "uint", + "minimum": 0 + }, + { + "title": "MergedRest", + "description": "All remaining axes, combined into one", + "type": "object", + "properties": {}, + "additionalProperties": false + } + ] + } + }, + { + "title": "AllRest", + "description": "All remaining axes, each in a separate single-axis group", + "type": "object", + "properties": {}, + "additionalProperties": false + } + ] + }, + "description": "The permutation of the axes that is applied on encoding.\n\nThe permutation is given as a list of axis groups, where each group\ncorresponds to one encoded output axis that may consist of several\ndecoded input axes. For instance, `[[0], [1, 2]]` flattens a three-\ndimensional array into a two-dimensional one by combining the second and\nthird axes.\n\nThe permutation also allows specifying a special catch-all remaining\naxes marker:\n- `[[0], {}]` moves the second axis to be the first and appends all\n other axes afterwards, i.e. the encoded array has the same number\n of axes as the input array\n- `[[0], [{}]]` in contrast collapses all other axes into one, i.e.\n the encoded array is two-dimensional" + }, + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "1.0.0" + } + }, + "required": [ + "axes" + ], + "description": "Codec to swizzle/swap the axes of an array and reshape it.\n\nThis codec does not store metadata about the original shape of the array.\nSince axes that have been combined during encoding cannot be split without\nfurther information, decoding may fail if an output array is not provided.\n\nSwizzling axes is always supported since no additional information about the\narray's shape is required to reconstruct it.", + "title": "SwizzleReshapeCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/codecs/swizzle-reshape/tests/schema.rs b/codecs/swizzle-reshape/tests/schema.rs new file mode 100644 index 000000000..418dbf1c2 --- /dev/null +++ b/codecs/swizzle-reshape/tests/schema.rs @@ -0,0 +1,21 @@ +#![expect(missing_docs)] + +use ::{ndarray as _, schemars as _, serde as _, serde_json as _, thiserror as _}; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_swizzle_reshape::SwizzleReshapeCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + #[expect(clippy::manual_assert, clippy::panic)] + if schema != include_str!("schema.json") { + panic!("SwizzleReshape schema has changed\n===\n{schema}\n==="); + } +} diff --git a/codecs/sz3/src/lib.rs b/codecs/sz3/src/lib.rs index e7c76ce0a..5c1016835 100644 --- a/codecs/sz3/src/lib.rs +++ b/codecs/sz3/src/lib.rs @@ -115,6 +115,7 @@ pub enum Sz3ErrorBound { /// SZ3 predictor #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] +#[schemars(inline)] pub enum Sz3Predictor { /// Interpolation #[serde(rename = "interpolation")] diff --git a/codecs/tthresh/src/lib.rs b/codecs/tthresh/src/lib.rs index 0eceb5de3..b7407f484 100644 --- a/codecs/tthresh/src/lib.rs +++ b/codecs/tthresh/src/lib.rs @@ -302,6 +302,10 @@ impl<'de> Deserialize<'de> for NonNegative { } impl JsonSchema for NonNegative { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("NonNegativeF64") } diff --git a/codecs/zfp-classic/src/lib.rs b/codecs/zfp-classic/src/lib.rs index 47522be98..a49e4c9ca 100644 --- a/codecs/zfp-classic/src/lib.rs +++ b/codecs/zfp-classic/src/lib.rs @@ -115,6 +115,7 @@ pub enum ZfpCompressionMode { } #[derive(Copy, Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] +#[schemars(inline)] /// ZFP non-finite values mode pub enum ZfpNonFiniteValuesMode { /// Deny compressing non-finite values, i.e. return an error. diff --git a/codecs/zfp/src/lib.rs b/codecs/zfp/src/lib.rs index cda53b053..3d315fd60 100644 --- a/codecs/zfp/src/lib.rs +++ b/codecs/zfp/src/lib.rs @@ -120,6 +120,7 @@ pub enum ZfpCompressionMode { } #[derive(Copy, Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] +#[schemars(inline)] /// ZFP non-finite values mode pub enum ZfpNonFiniteValuesMode { /// Deny compressing non-finite values, i.e. return an error. diff --git a/crates/numcodecs-wasm-guest/Cargo.toml b/crates/numcodecs-wasm-guest/Cargo.toml index 31df535e3..4a1707809 100644 --- a/crates/numcodecs-wasm-guest/Cargo.toml +++ b/crates/numcodecs-wasm-guest/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "numcodecs-wasm-guest" -version = "0.3.0" +version = "0.3.1" edition = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/crates/numcodecs-wasm-guest/src/lib.rs b/crates/numcodecs-wasm-guest/src/lib.rs index 90880ff6c..0ed6e394c 100644 --- a/crates/numcodecs-wasm-guest/src/lib.rs +++ b/crates/numcodecs-wasm-guest/src/lib.rs @@ -27,8 +27,7 @@ use numcodecs::StaticCodec; #[cfg(target_arch = "wasm32")] use ::{ - numcodecs::{Codec, StaticCodec}, - schemars::schema_for, + numcodecs::{Codec, DynCodecType, StaticCodec, StaticCodecType}, serde::Deserialize, }; @@ -104,8 +103,9 @@ impl wit::Guest for T { } fn codec_config_schema() -> wit::JsonSchema { - schema_for!(::Config<'static>) - .as_value() + StaticCodecType::::of() + .codec_config_schema() + .to_value() .to_string() } } diff --git a/crates/numcodecs/Cargo.toml b/crates/numcodecs/Cargo.toml index daf48e7b5..3f9456d5f 100644 --- a/crates/numcodecs/Cargo.toml +++ b/crates/numcodecs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "numcodecs" -version = "0.3.1" +version = "0.3.2" edition = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/crates/numcodecs/src/codec.rs b/crates/numcodecs/src/codec.rs index 0e144615c..6be95ee63 100644 --- a/crates/numcodecs/src/codec.rs +++ b/crates/numcodecs/src/codec.rs @@ -1,6 +1,10 @@ use std::{borrow::Cow, error::Error, fmt, marker::PhantomData}; -use schemars::{JsonSchema, Schema, SchemaGenerator, generate::SchemaSettings, json_schema}; +use schemars::{ + JsonSchema, Schema, SchemaGenerator, + generate::{Contract, SchemaSettings}, + json_schema, +}; use semver::{Version, VersionReq}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::Value; @@ -152,9 +156,9 @@ impl DynCodecType for StaticCodecType { fn codec_config_schema(&self) -> Schema { let mut settings = SchemaSettings::draft2020_12(); - // TODO: perhaps this could be done as a more generally applicable - // transformation instead - settings.inline_subschemas = true; + settings.inline_subschemas = false; + settings.contract = Contract::Deserialize; + settings.untagged_enum_variant_titles = true; settings .into_generator() .into_root_schema_for::>() @@ -392,6 +396,10 @@ impl<'de, const MAJOR: u64, const MINOR: u64, const PATCH: u64> Deserialize<'de> impl JsonSchema for StaticCodecVersion { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("StaticCodecVersion") } From caeb54e32a4ca5479492667662964d3729f37e76 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 23 Apr 2026 10:38:59 +0300 Subject: [PATCH 52/55] Remove extraneous schemars dependency in numcodecs-wasm-guest --- crates/numcodecs-wasm-guest/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/numcodecs-wasm-guest/Cargo.toml b/crates/numcodecs-wasm-guest/Cargo.toml index 4a1707809..4223e42f3 100644 --- a/crates/numcodecs-wasm-guest/Cargo.toml +++ b/crates/numcodecs-wasm-guest/Cargo.toml @@ -19,7 +19,6 @@ wit-bindgen = { workspace = true, features = ["macros", "realloc"] } [target.'cfg(target_arch = "wasm32")'.dependencies] format_serde_error = { workspace = true, features = ["serde_json"] } ndarray = { workspace = true, features = ["std"] } -schemars = { workspace = true } serde = { workspace = true } serde_json = { workspace = true, features = ["std"] } thiserror = { workspace = true } From c69c1de0e484e7762e51efbca3b1740061618bb2 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 30 Apr 2026 15:54:37 +0300 Subject: [PATCH 53/55] add failing config test --- codecs/pressio/tests/config.rs | 69 ++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 codecs/pressio/tests/config.rs diff --git a/codecs/pressio/tests/config.rs b/codecs/pressio/tests/config.rs new file mode 100644 index 000000000..0eacd8eb6 --- /dev/null +++ b/codecs/pressio/tests/config.rs @@ -0,0 +1,69 @@ +#![expect(missing_docs, clippy::unwrap_used)] + +use ::{ + fragile as _, libpressio as _, ndarray as _, schemars as _, serde as _, serde_json as _, + serde_ndim as _, thiserror as _, +}; + +use numcodecs::StaticCodec; +use numcodecs_pressio::PressioCodec; +use serde::Deserialize; +use serde_json::json; + +#[test] +#[should_panic(expected = "missing field `compressor_id`")] +fn empty_config() { + let _ = PressioCodec::from_config(Deserialize::deserialize(json!({})).unwrap()); +} + +#[test] +#[should_panic(expected = "invalid compressor id ???, choose one of")] +fn invalid_compressor_id() { + let _ = PressioCodec::from_config( + Deserialize::deserialize(json!({ + "compressor_id": "???", + })) + .unwrap(), + ); +} + +#[test] +#[should_panic(expected = "unknown compressor configuration option: `abc`, use one of")] +fn unknown_compressor_config() { + let _ = PressioCodec::from_config( + Deserialize::deserialize(json!({ + "compressor_id": "linear_quantizer", + "compressor_config": { + "abc": 42, + } + })) + .unwrap(), + ); +} + +#[test] +#[should_panic(expected = "failed to cast option `pressio:abs`")] +fn option_cast_failure() { + let _ = PressioCodec::from_config( + Deserialize::deserialize(json!({ + "compressor_id": "linear_quantizer", + "compressor_config": { + "pressio:abs": "abc", + }, + })) + .unwrap(), + ); +} + +#[test] +fn array_data_option() { + let _ = PressioCodec::from_config( + Deserialize::deserialize(json!({ + "compressor_id": "mask_interpolation", + "compressor_config": { + "mask_interpolation:mask": [true, false], + }, + })) + .unwrap(), + ); +} From cd298dbbc7e5bae528fac7d133189aa648adaac1 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 30 Apr 2026 21:01:45 +0300 Subject: [PATCH 54/55] Fix pressio bool data array --- Cargo.toml | 2 +- codecs/pressio/tests/config.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b625fa8b6..615bbb2c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,7 +119,7 @@ schemars = { version = "1.0.3", default-features = false } scratch = { version = "1.0", default-features = false } semver = { version = "1.0.23", default-features = false } serde = { version = "1.0.218", default-features = false } -serde-ndim = { version = "=2.1.0", default-features = false } +serde-ndim = { version = "=2.1.1", git = "https://github.com/juntyr/serde-ndim.git", rev = "060c889", default-features = false } serde-transcode = { version = "1.1", default-features = false } serde_json = { version = "1.0.140", default-features = false } serde_repr = { version = "0.1.5", default-features = false } diff --git a/codecs/pressio/tests/config.rs b/codecs/pressio/tests/config.rs index 0eacd8eb6..e87377ca4 100644 --- a/codecs/pressio/tests/config.rs +++ b/codecs/pressio/tests/config.rs @@ -56,12 +56,12 @@ fn option_cast_failure() { } #[test] -fn array_data_option() { +fn bool_array_data_option() { let _ = PressioCodec::from_config( Deserialize::deserialize(json!({ "compressor_id": "mask_interpolation", "compressor_config": { - "mask_interpolation:mask": [true, false], + "mask_interpolation:mask": [[true, false], [false, true]], }, })) .unwrap(), From ef7bfaa454e5bd67735645cf50f9e36f7088ba79 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 30 Apr 2026 21:14:13 +0300 Subject: [PATCH 55/55] Prepare manual v0.1.0-beta.1 release --- codecs/pressio/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index 87a49a490..cf76dbcae 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "numcodecs-pressio" -version = "0.1.0" +version = "0.1.0-beta.1" edition = { workspace = true } authors = { workspace = true } repository = { workspace = true }