[DRAFT] Implement numcodecs-registry on wasm-guest

Cargo.toml

@@ -3,6 +3,7 @@ resolver = "2"
 members = [
     "crates/numcodecs",
     "crates/numcodecs-python",
+    "crates/numcodecs-registry",
     "crates/numcodecs-wasm-builder",
     "crates/numcodecs-wasm-logging",
     "crates/numcodecs-wasm-guest",
@@ -21,6 +22,7 @@ members = [
     "codecs/lc",
     "codecs/linear-quantize",
     "codecs/log",
+    "codecs/onion",
     "codecs/pco",
     "codecs/qpet-sperr",
     "codecs/random-projection",
@@ -49,6 +51,7 @@ rust-version = "1.87"
 # workspace-internal numcodecs crates
 numcodecs = { version = "0.3.1", path = "crates/numcodecs", default-features = false }
 numcodecs-python = { version = "0.7.1", path = "crates/numcodecs-python", default-features = false }
+numcodecs-registry = { version = "0.1", path = "crates/numcodecs-registry", default-features = false }
 numcodecs-wasm-builder = { version = "0.2", path = "crates/numcodecs-wasm-builder", default-features = false }
 numcodecs-wasm-guest = { version = "0.3", path = "crates/numcodecs-wasm-guest", default-features = false }
 numcodecs-wasm-host = { version = "0.2", path = "crates/numcodecs-wasm-host", default-features = false }
@@ -68,6 +71,7 @@ numcodecs-jpeg2000 = { version = "0.3", path = "codecs/jpeg2000", default-featur
 numcodecs-lc = { version = "0.1", path = "codecs/lc", default-features = false }
 numcodecs-linear-quantize = { version = "0.5", path = "codecs/linear-quantize", default-features = false }
 numcodecs-log = { version = "0.5", path = "codecs/log", default-features = false }
+numcodecs-onion = { version = "0.1", path = "codecs/onion", default-features = false }
 numcodecs-pco = { version = "0.4", path = "codecs/pco", default-features = false }
 numcodecs-qpet-sperr = { version = "0.2.2", path = "codecs/qpet-sperr", default-features = false }
 numcodecs-random-projection = { version = "0.4", path = "codecs/random-projection", default-features = false }
@@ -90,6 +94,7 @@ burn = { version = "0.18", default-features = false }
 clap = { version = "4.6", default-features = false }
 convert_case = { version = "0.8", default-features = false }
 ebcc = { version = "0.3.0-alpha", default-features = false }
+erased-serde = { version = "0.4", default-features = false }
 format_serde_error = { version = "0.3", default-features = false }
 indexmap = { version = "2.10", default-features = false }
 itertools = { version = "0.14", default-features = false }

codecs/onion/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "numcodecs-onion"
+version = "0.1.0"
+edition = { workspace = true }
+authors = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+rust-version = { workspace = true }
+
+description = "Onion identity meta-codec implementation for the numcodecs API"
+readme = "README.md"
+categories = ["compression", "encoding"]
+keywords = ["identity", "numcodecs", "compression", "encoding", "meta"]
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+numcodecs = { workspace = true }
+numcodecs-registry = { workspace = true }
+schemars = { workspace = true, features = ["derive", "preserve_order"] }
+serde = { workspace = true, features = ["std", "derive"] }
+thiserror = { workspace = true }
+
+[lints]
+workspace = true

codecs/onion/LICENSE

@@ -0,0 +1 @@
+../../LICENSE

codecs/onion/README.md

@@ -0,0 +1,38 @@
+[![CI Status]][workflow] [![MSRV]][repo] [![Latest Version]][crates.io] [![PyPi Release]][pypi] [![Rust Doc Crate]][docs.rs] [![Rust Doc Main]][docs] [![Read the Docs]][rtdocs]
+
+[CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main
+[workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain
+
+[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue
+[repo]: https://github.com/juntyr/numcodecs-rs
+
+[Latest Version]: https://img.shields.io/crates/v/numcodecs-onion
+[crates.io]: https://crates.io/crates/numcodecs-onion
+
+[PyPi Release]: https://img.shields.io/pypi/v/numcodecs-wasm-onion.svg
+[pypi]: https://pypi.python.org/pypi/numcodecs-wasm-onion
+
+[Rust Doc Crate]: https://img.shields.io/docsrs/numcodecs-onion
+[docs.rs]: https://docs.rs/numcodecs-onion/
+
+[Rust Doc Main]: https://img.shields.io/badge/docs-main-blue
+[docs]: https://juntyr.github.io/numcodecs-rs/numcodecs_onion
+
+[Read the Docs]: https://img.shields.io/readthedocs/numcodecs-wasm?label=readthedocs
+[rtdocs]: https://numcodecs-wasm.readthedocs.io/en/stable/api/numcodecs_wasm_onion/
+
+# numcodecs-onion
+
+Onion identity meta-codec implementation for the [`numcodecs`] API.
+
+[`numcodecs`]: https://docs.rs/numcodecs/0.2/numcodecs/
+
+## License
+
+Licensed under the Mozilla Public License, Version 2.0 ([LICENSE](LICENSE) or https://www.mozilla.org/en-US/MPL/2.0/).
+
+## Funding
+
+The `numcodecs-onion` crate has been developed as part of [ESiWACE3](https://www.esiwace.eu), the third phase of the Centre of Excellence in Simulation of Weather and Climate in Europe.
+
+Funded by the European Union. This work has received funding from the European High Performance Computing Joint Undertaking (JU) under grant agreement No 101093054.

codecs/onion/src/lib.rs

@@ -0,0 +1,86 @@
+//! [![CI Status]][workflow] [![MSRV]][repo] [![Latest Version]][crates.io] [![Rust Doc Crate]][docs.rs] [![Rust Doc Main]][docs]
+//!
+//! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main
+//! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain
+//!
+//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue
+//! [repo]: https://github.com/juntyr/numcodecs-rs
+//!
+//! [Latest Version]: https://img.shields.io/crates/v/numcodecs-onion
+//! [crates.io]: https://crates.io/crates/numcodecs-onion
+//!
+//! [Rust Doc Crate]: https://img.shields.io/docsrs/numcodecs-onion
+//! [docs.rs]: https://docs.rs/numcodecs-onion/
+//!
+//! [Rust Doc Main]: https://img.shields.io/badge/docs-main-blue
+//! [docs]: https://juntyr.github.io/numcodecs-rs/numcodecs_onion
+//!
+//! Onion identity meta-codec implementation for the [`numcodecs`] API.
+
+use numcodecs::{
+    AnyArray, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, ErasedError, StaticCodec,
+    StaticCodecConfig, StaticCodecVersion,
+};
+use numcodecs_registry::GlobalErasedDynCodec;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+#[derive(Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(deny_unknown_fields)]
+/// Onion identity meta-codec which wraps an existing codec and passes the
+/// inputs to and outputs from it unchanged during encoding and decoding.
+pub struct OnionCodec {
+    /// The configuration of the wrapped codec.
+    pub codec: GlobalErasedDynCodec,
+    /// The codec's encoding format version. Do not provide this parameter explicitly.
+    #[serde(default, rename = "_version")]
+    pub version: StaticCodecVersion<1, 0, 0>,
+}
+
+impl Codec for OnionCodec {
+    type Error = OnionCodecError;
+
+    fn encode(&self, data: AnyCowArray) -> Result<AnyArray, Self::Error> {
+        self.codec
+            .encode(data)
+            .map_err(|err| OnionCodecError { error: err })
+    }
+
+    fn decode(&self, encoded: AnyCowArray) -> Result<AnyArray, Self::Error> {
+        self.codec
+            .decode(encoded)
+            .map_err(|err| OnionCodecError { error: err })
+    }
+
+    fn decode_into(
+        &self,
+        encoded: AnyArrayView,
+        decoded: AnyArrayViewMut,
+    ) -> Result<(), Self::Error> {
+        self.codec
+            .decode_into(encoded, decoded)
+            .map_err(|err| OnionCodecError { error: err })
+    }
+}
+
+impl StaticCodec for OnionCodec {
+    const CODEC_ID: &'static str = "onion.rs";
+
+    type Config<'de> = Self;
+
+    fn from_config(config: Self::Config<'_>) -> Self {
+        config
+    }
+
+    fn get_config(&self) -> StaticCodecConfig<'_, Self> {
+        StaticCodecConfig::from(self)
+    }
+}
+
+#[derive(Debug, Error)]
+/// Errors that may occur when applying the [`OnionCodec`].
+#[error(transparent)]
+pub struct OnionCodecError {
+    error: ErasedError,
+}

codecs/onion/tests/schema.json

@@ -0,0 +1,34 @@
+{
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "codec": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "type": "string",
+          "description": "The `codec_id` of the codec, which is looked up in the global\nregistry."
+        }
+      },
+      "required": [
+        "id"
+      ],
+      "description": "The configuration of the wrapped codec.",
+      "additionalProperties": {
+        "type": "object"
+      }
+    },
+    "_version": {
+      "type": "string",
+      "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$",
+      "description": "The codec's encoding format version. Do not provide this parameter explicitly.",
+      "default": "1.0.0"
+    }
+  },
+  "required": [
+    "codec"
+  ],
+  "description": "Onion identity meta-codec which wraps an existing codec and passes the\ninputs to and outputs from it unchanged during encoding and decoding.",
+  "title": "OnionCodec",
+  "$schema": "https://json-schema.org/draft/2020-12/schema"
+}

codecs/onion/tests/schema.rs

@@ -0,0 +1,20 @@
+#![expect(missing_docs)]
+
+use ::{numcodecs_registry as _, schemars as _, serde as _, thiserror as _};
+
+use numcodecs::{DynCodecType, StaticCodecType};
+use numcodecs_onion::OnionCodec;
+
+#[test]
+fn schema() {
+    let schema = format!(
+        "{:#}",
+        StaticCodecType::<OnionCodec>::of()
+            .codec_config_schema()
+            .to_value()
+    );
+
+    if schema != include_str!("schema.json") {
+        panic!("Onion schema has changed\n===\n{schema}\n===");
+    }
+}

crates/numcodecs-python/Cargo.toml

@@ -18,6 +18,7 @@ keywords = ["numcodecs", "compression", "encoding", "python", "pyo3"]
 convert_case = { workspace = true }
 ndarray = { workspace = true }
 numcodecs = { workspace = true }
+numcodecs-registry = { workspace = true }
 numpy = { workspace = true }
 pyo3 = { workspace = true }
 pyo3-error = { workspace = true }

crates/numcodecs-python/src/registry.rs

@@ -1,13 +1,16 @@
+use numcodecs::{DynCodec, ErasedDynCodec};
+use numcodecs_registry::Registry;
 use pyo3::{prelude::*, sync::PyOnceLock, types::PyDict};
+use pythonize::Pythonizer;
+use serde::Deserializer;
+use serde_transcode::transcode;
 
 #[expect(unused_imports)] // FIXME: use expect, only used in docs
 use crate::PyCodecClassMethods;
-use crate::{PyCodec, PyCodecClass};
+use crate::{PyCodec, PyCodecAdapter, PyCodecClass};
 
 /// Dynamic registry of codec classes.
-pub struct PyCodecRegistry {
-    _private: (),
-}
+pub struct PyCodecRegistry;
 
 impl PyCodecRegistry {
     /// Instantiate a codec from a configuration dictionary.
@@ -56,3 +59,38 @@ impl PyCodecRegistry {
         Ok(())
     }
 }
+
+impl Registry for PyCodecRegistry {
+    type Error = PyErr;
+
+    fn get_codec<'de, D: Deserializer<'de>>(
+        &self,
+        config: D,
+    ) -> Result<ErasedDynCodec, Self::Error> {
+        Python::attach(|py| {
+            let config = transcode(config, Pythonizer::new(py))?;
+            let config: Bound<PyDict> = config.extract()?;
+
+            let codec = Self::get_codec(config.as_borrowed())?;
+            let codec = PyCodecAdapter::from_codec(codec)?;
+
+            Ok(ErasedDynCodec::new(codec))
+        })
+    }
+
+    fn get_codec_typed<'de, T: DynCodec, D: Deserializer<'de>>(
+        &self,
+        config: D,
+    ) -> Result<Option<T>, Self::Error> {
+        Python::attach(|py| {
+            let config = transcode(config, Pythonizer::new(py))?;
+            let config: Bound<PyDict> = config.extract()?;
+
+            let codec = Self::get_codec(config.as_borrowed())?;
+            // clone is necessary since we cannot move out of a PyCodec
+            let codec = PyCodecAdapter::with_downcast(py, &codec, |codec: &T| codec.clone());
+
+            Ok(codec)
+        })
+    }
+}

crates/numcodecs-python/tests/schema.rs

@@ -23,7 +23,7 @@ fn collect_schemas() -> Result<(), PyErr> {
 
             println!(
                 "{codec_id}: {:#}",
-                codec_ty.codec_config_schema().as_value()
+                codec_ty.codec_config_schema().to_value()
             );
         }
 

crates/numcodecs-registry/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "numcodecs-registry"
+version = "0.1.0"
+edition = { workspace = true }
+authors = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+rust-version = { workspace = true }
+
+description = "registries for numcodecs codecs"
+readme = "README.md"
+categories = ["compression", "encoding"]
+keywords = ["numcodecs", "registry", "compression", "encoding"]
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+erased-serde = { workspace = true, features = ["std"] }
+numcodecs = { workspace = true }
+schemars = { workspace = true, features = ["derive"] }
+serde = { workspace = true }
+thiserror = { workspace = true }
+
+[lints]
+workspace = true

crates/numcodecs-registry/LICENSE

@@ -0,0 +1 @@
+../../LICENSE

crates/numcodecs-registry/README.md

@@ -0,0 +1,32 @@
+[![CI Status]][workflow] [![MSRV]][repo] [![Latest Version]][crates.io] [![Rust Doc Crate]][docs.rs] [![Rust Doc Main]][docs]
+
+[CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main
+[workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain
+
+[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue
+[repo]: https://github.com/juntyr/numcodecs-rs
+
+[Latest Version]: https://img.shields.io/crates/v/numcodecs-registry
+[crates.io]: https://crates.io/crates/numcodecs-registry
+
+[Rust Doc Crate]: https://img.shields.io/docsrs/numcodecs-registry
+[docs.rs]: https://docs.rs/numcodecs-registry/
+
+[Rust Doc Main]: https://img.shields.io/badge/docs-main-blue
+[docs]: https://juntyr.github.io/numcodecs-rs/numcodecs_registry
+
+# numcodecs-registry
+
+Registries for compression codecs implementing the [`numcodecs`] API.
+
+[`numcodecs`]: https://numcodecs.readthedocs.io/en/stable/
+
+## License
+
+Licensed under the Mozilla Public License, Version 2.0 ([LICENSE](LICENSE) or https://www.mozilla.org/en-US/MPL/2.0/).
+
+## Funding
+
+The `numcodecs-registry` crate has been developed as part of [ESiWACE3](https://www.esiwace.eu), the third phase of the Centre of Excellence in Simulation of Weather and Climate in Europe.
+
+Funded by the European Union. This work has received funding from the European High Performance Computing Joint Undertaking (JU) under grant agreement No 101093054.