diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index effd04b26..512152833 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -357,7 +357,7 @@ jobs: rm .python-version uv python install ${{ matrix.python }} uv sync && uv pip install . - uv run python3 -m numcodecs_wasm_materialize "^${{ matrix.codec }}$" --local + uv run python3 -m numcodecs_wasm_materialize "^${{ matrix.codec }}$" --local --debug - name: Upload the ${{ matrix.codec }} WASM codec if: ${{ matrix.lock == 'Cargo.lock' }} diff --git a/Cargo.toml b/Cargo.toml index d873c7bd5..32b9ca073 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,7 +80,7 @@ numcodecs-round = { version = "0.5", path = "codecs/round", default-features = f numcodecs-sperr = { version = "0.2.3", path = "codecs/sperr", default-features = false } numcodecs-stochastic-rounding = { version = "0.2", path = "codecs/stochastic-rounding", default-features = false } numcodecs-swizzle-reshape = { version = "0.4", path = "codecs/swizzle-reshape", default-features = false } -numcodecs-sz3 = { version = "0.8.4", path = "codecs/sz3", default-features = false } +numcodecs-sz3 = { version = "0.7.1", path = "codecs/sz3", default-features = false } numcodecs-tthresh = { version = "0.3.2", path = "codecs/tthresh", default-features = false } numcodecs-uniform-noise = { version = "0.4", path = "codecs/uniform-noise", default-features = false } numcodecs-zfp = { version = "0.6", path = "codecs/zfp", default-features = false } @@ -125,7 +125,7 @@ serde_json = { version = "1.0.140", default-features = false } serde_repr = { version = "0.1.5", default-features = false } simple_logger = { version = "5.0", default-features = false } sperr = { version = "0.2.1", default-features = false } -sz3 = { version = "0.4.3", default-features = false } +sz3 = { version = "0.3", default-features = false } thiserror = { version = "2.0.17", default-features = false } tthresh = { version = "0.1", default-features = false } twofloat = { version = "0.8", default-features = false } diff --git a/codecs/sz3/Cargo.toml b/codecs/sz3/Cargo.toml index 3e043d895..c528a28db 100644 --- a/codecs/sz3/Cargo.toml +++ b/codecs/sz3/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "numcodecs-sz3" -version = "0.8.4" +version = "0.7.1" edition = { workspace = true } authors = { workspace = true } repository = { workspace = true } @@ -26,7 +26,6 @@ thiserror = { workspace = true } zstd-sys = { workspace = true, features = ["no_wasm_shim"] } [dev-dependencies] -num-traits = { workspace = true, features = ["std"] } serde_json = { workspace = true, features = ["std"] } [lints] diff --git a/codecs/sz3/src/lib.rs b/codecs/sz3/src/lib.rs index 444f45877..f95de9fc4 100644 --- a/codecs/sz3/src/lib.rs +++ b/codecs/sz3/src/lib.rs @@ -21,10 +21,10 @@ use std::{borrow::Cow, fmt}; -use ndarray::{Array, Array1, ArrayBase, ArrayViewMut, Data, Dimension, IxDyn, ShapeError}; +use ndarray::{Array, Array1, ArrayBase, Data, Dimension, ShapeError}; use numcodecs::{ AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray, - ArrayDType, ArrayDataMutExt, Codec, StaticCodec, StaticCodecConfig, StaticCodecVersion, + Codec, StaticCodec, StaticCodecConfig, StaticCodecVersion, }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -37,7 +37,7 @@ use ::zstd_sys as _; #[cfg(test)] use ::serde_json as _; -type Sz3CodecVersion = StaticCodecVersion<0, 2, 0>; +type Sz3CodecVersion = StaticCodecVersion<0, 1, 0>; #[derive(Clone, Serialize, Deserialize, JsonSchema)] // serde cannot deny unknown fields because of the flatten @@ -116,38 +116,68 @@ pub enum Sz3ErrorBound { #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub enum Sz3Predictor { - /// Interpolation - #[serde(rename = "interpolation")] - Interpolation, - /// Interpolation + Lorenzo predictor - #[serde(rename = "interpolation-lorenzo")] - InterpolationLorenzo, + /// Linear interpolation + #[serde(rename = "linear-interpolation")] + LinearInterpolation, + /// Cubic interpolation + #[serde(rename = "cubic-interpolation")] + CubicInterpolation, + /// Linear interpolation + Lorenzo predictor + #[serde(rename = "linear-interpolation-lorenzo")] + LinearInterpolationLorenzo, + /// Cubic interpolation + Lorenzo predictor + #[serde(rename = "cubic-interpolation-lorenzo")] + CubicInterpolationLorenzo, /// 1st order regression #[serde(rename = "regression")] Regression, + /// 2nd order regression + #[serde(rename = "regression2")] + RegressionSecondOrder, + /// 1st+2nd order regression + #[serde(rename = "regression-regression2")] + RegressionFirstSecondOrder, /// 2nd order Lorenzo predictor #[serde(rename = "lorenzo2")] LorenzoSecondOrder, + /// 2nd order Lorenzo predictor + 2nd order regression + #[serde(rename = "lorenzo2-regression2")] + LorenzoSecondOrderRegressionSecondOrder, /// 2nd order Lorenzo predictor + 1st order regression #[serde(rename = "lorenzo2-regression")] LorenzoSecondOrderRegression, + /// 2nd order Lorenzo predictor + 1st order regression + #[serde(rename = "lorenzo2-regression-regression2")] + LorenzoSecondOrderRegressionFirstSecondOrder, /// 1st order Lorenzo predictor #[serde(rename = "lorenzo")] Lorenzo, + /// 1st order Lorenzo predictor + 2nd order regression + #[serde(rename = "lorenzo-regression2")] + LorenzoRegressionSecondOrder, /// 1st order Lorenzo predictor + 1st order regression #[serde(rename = "lorenzo-regression")] LorenzoRegression, + /// 1st order Lorenzo predictor + 1st and 2nd order regression + #[serde(rename = "lorenzo-regression-regression2")] + LorenzoRegressionFirstSecondOrder, /// 1st+2nd order Lorenzo predictor #[serde(rename = "lorenzo-lorenzo2")] LorenzoFirstSecondOrder, + /// 1st+2nd order Lorenzo predictor + 2nd order regression + #[serde(rename = "lorenzo-lorenzo2-regression2")] + LorenzoFirstSecondOrderRegressionSecondOrder, /// 1st+2nd order Lorenzo predictor + 1st order regression #[serde(rename = "lorenzo-lorenzo2-regression")] LorenzoFirstSecondOrderRegression, + /// 1st+2nd order Lorenzo predictor + 1st+2nd order regression + #[serde(rename = "lorenzo-lorenzo2-regression-regression2")] + LorenzoFirstSecondOrderRegressionFirstSecondOrder, } #[expect(clippy::unnecessary_wraps)] const fn default_predictor() -> Option { - Some(Sz3Predictor::InterpolationLorenzo) + Some(Sz3Predictor::CubicInterpolationLorenzo) } impl Codec for Sz3Codec { @@ -155,34 +185,10 @@ impl Codec for Sz3Codec { fn encode(&self, data: AnyCowArray) -> Result { match data { - AnyCowArray::U8(data) => Ok(AnyArray::U8( - Array1::from(compress(data, self.predictor.as_ref(), &self.error_bound)?) - .into_dyn(), - )), - AnyCowArray::I8(data) => Ok(AnyArray::U8( - Array1::from(compress(data, self.predictor.as_ref(), &self.error_bound)?) - .into_dyn(), - )), - AnyCowArray::U16(data) => Ok(AnyArray::U8( - Array1::from(compress(data, self.predictor.as_ref(), &self.error_bound)?) - .into_dyn(), - )), - AnyCowArray::I16(data) => Ok(AnyArray::U8( - Array1::from(compress(data, self.predictor.as_ref(), &self.error_bound)?) - .into_dyn(), - )), - AnyCowArray::U32(data) => Ok(AnyArray::U8( - Array1::from(compress(data, self.predictor.as_ref(), &self.error_bound)?) - .into_dyn(), - )), AnyCowArray::I32(data) => Ok(AnyArray::U8( Array1::from(compress(data, self.predictor.as_ref(), &self.error_bound)?) .into_dyn(), )), - AnyCowArray::U64(data) => Ok(AnyArray::U8( - Array1::from(compress(data, self.predictor.as_ref(), &self.error_bound)?) - .into_dyn(), - )), AnyCowArray::I64(data) => Ok(AnyArray::U8( Array1::from(compress(data, self.predictor.as_ref(), &self.error_bound)?) .into_dyn(), @@ -218,21 +224,11 @@ impl Codec for Sz3Codec { fn decode_into( &self, encoded: AnyArrayView, - decoded: AnyArrayViewMut, + mut decoded: AnyArrayViewMut, ) -> Result<(), Self::Error> { - let AnyArrayView::U8(encoded) = encoded else { - return Err(Sz3CodecError::EncodedDataNotBytes { - dtype: encoded.dtype(), - }); - }; - - if !matches!(encoded.shape(), [_]) { - return Err(Sz3CodecError::EncodedDataNotOneDimensional { - shape: encoded.shape().to_vec(), - }); - } + let decoded_in = self.decode(encoded.cow())?; - decompress_into(&AnyArrayView::U8(encoded).as_bytes(), decoded) + Ok(decoded.assign(&decoded_in)?) } } @@ -300,12 +296,6 @@ pub enum Sz3CodecError { /// Opaque source error source: Sz3HeaderError, }, - /// [`Sz3Codec`] failed to decode the data - #[error("Sz3 failed to decode the data")] - Sz3DecodeFailed { - /// Opaque source error - source: Sz3CodingError, - }, /// [`Sz3Codec`] decoded an invalid array shape header which does not fit /// the decoded data #[error("Sz3 decoded an invalid array shape header which does not fit the decoded data")] @@ -422,42 +412,150 @@ pub fn compress, D: Dimension>( }; let mut config = sz3::Config::new(error_bound); + // configure the interpolation mode, if necessary + let interpolation = match predictor { + Some(Sz3Predictor::LinearInterpolation | Sz3Predictor::LinearInterpolationLorenzo) => { + Some(sz3::InterpolationAlgorithm::Linear) + } + Some(Sz3Predictor::CubicInterpolation | Sz3Predictor::CubicInterpolationLorenzo) => { + Some(sz3::InterpolationAlgorithm::Cubic) + } + Some( + Sz3Predictor::Regression + | Sz3Predictor::RegressionSecondOrder + | Sz3Predictor::RegressionFirstSecondOrder + | Sz3Predictor::LorenzoSecondOrder + | Sz3Predictor::LorenzoSecondOrderRegressionSecondOrder + | Sz3Predictor::LorenzoSecondOrderRegression + | Sz3Predictor::LorenzoSecondOrderRegressionFirstSecondOrder + | Sz3Predictor::Lorenzo + | Sz3Predictor::LorenzoRegressionSecondOrder + | Sz3Predictor::LorenzoRegression + | Sz3Predictor::LorenzoRegressionFirstSecondOrder + | Sz3Predictor::LorenzoFirstSecondOrder + | Sz3Predictor::LorenzoFirstSecondOrderRegressionSecondOrder + | Sz3Predictor::LorenzoFirstSecondOrderRegression + | Sz3Predictor::LorenzoFirstSecondOrderRegressionFirstSecondOrder, + ) + | None => None, + }; + if let Some(interpolation) = interpolation { + config = config.interpolation_algorithm(interpolation); + } + // configure the predictor (compression algorithm) let predictor = match predictor { - Some(Sz3Predictor::Interpolation) => sz3::CompressionAlgorithm::Interpolation, - Some(Sz3Predictor::InterpolationLorenzo) => sz3::CompressionAlgorithm::InterpolationLorenzo, + Some(Sz3Predictor::LinearInterpolation | Sz3Predictor::CubicInterpolation) => { + sz3::CompressionAlgorithm::Interpolation + } + Some( + Sz3Predictor::LinearInterpolationLorenzo | Sz3Predictor::CubicInterpolationLorenzo, + ) => sz3::CompressionAlgorithm::InterpolationLorenzo, + Some(Sz3Predictor::RegressionSecondOrder) => sz3::CompressionAlgorithm::LorenzoRegression { + lorenzo: false, + lorenzo_second_order: false, + regression: false, + regression_second_order: true, + prediction_dimension: None, + }, Some(Sz3Predictor::Regression) => sz3::CompressionAlgorithm::LorenzoRegression { lorenzo: false, lorenzo_second_order: false, regression: true, + regression_second_order: false, + prediction_dimension: None, }, + Some(Sz3Predictor::RegressionFirstSecondOrder) => { + sz3::CompressionAlgorithm::LorenzoRegression { + lorenzo: false, + lorenzo_second_order: false, + regression: true, + regression_second_order: true, + prediction_dimension: None, + } + } Some(Sz3Predictor::LorenzoSecondOrder) => sz3::CompressionAlgorithm::LorenzoRegression { lorenzo: false, lorenzo_second_order: true, regression: false, + regression_second_order: false, + prediction_dimension: None, }, + Some(Sz3Predictor::LorenzoSecondOrderRegressionSecondOrder) => { + sz3::CompressionAlgorithm::LorenzoRegression { + lorenzo: false, + lorenzo_second_order: true, + regression: false, + regression_second_order: true, + prediction_dimension: None, + } + } Some(Sz3Predictor::LorenzoSecondOrderRegression) => { sz3::CompressionAlgorithm::LorenzoRegression { lorenzo: false, lorenzo_second_order: true, regression: true, + regression_second_order: false, + prediction_dimension: None, + } + } + Some(Sz3Predictor::LorenzoSecondOrderRegressionFirstSecondOrder) => { + sz3::CompressionAlgorithm::LorenzoRegression { + lorenzo: false, + lorenzo_second_order: true, + regression: true, + regression_second_order: true, + prediction_dimension: None, } } Some(Sz3Predictor::Lorenzo) => sz3::CompressionAlgorithm::LorenzoRegression { lorenzo: true, lorenzo_second_order: false, regression: false, + regression_second_order: false, + prediction_dimension: None, }, + Some(Sz3Predictor::LorenzoRegressionSecondOrder) => { + sz3::CompressionAlgorithm::LorenzoRegression { + lorenzo: true, + lorenzo_second_order: false, + regression: false, + regression_second_order: true, + prediction_dimension: None, + } + } Some(Sz3Predictor::LorenzoRegression) => sz3::CompressionAlgorithm::LorenzoRegression { lorenzo: true, lorenzo_second_order: false, regression: true, + regression_second_order: false, + prediction_dimension: None, }, + Some(Sz3Predictor::LorenzoRegressionFirstSecondOrder) => { + sz3::CompressionAlgorithm::LorenzoRegression { + lorenzo: true, + lorenzo_second_order: false, + regression: true, + regression_second_order: true, + prediction_dimension: None, + } + } Some(Sz3Predictor::LorenzoFirstSecondOrder) => { sz3::CompressionAlgorithm::LorenzoRegression { lorenzo: true, lorenzo_second_order: true, regression: false, + regression_second_order: false, + prediction_dimension: None, + } + } + Some(Sz3Predictor::LorenzoFirstSecondOrderRegressionSecondOrder) => { + sz3::CompressionAlgorithm::LorenzoRegression { + lorenzo: true, + lorenzo_second_order: true, + regression: false, + regression_second_order: true, + prediction_dimension: None, } } Some(Sz3Predictor::LorenzoFirstSecondOrderRegression) => { @@ -465,45 +563,41 @@ pub fn compress, D: Dimension>( lorenzo: true, lorenzo_second_order: true, regression: true, + regression_second_order: false, + prediction_dimension: None, + } + } + Some(Sz3Predictor::LorenzoFirstSecondOrderRegressionFirstSecondOrder) => { + sz3::CompressionAlgorithm::LorenzoRegression { + lorenzo: true, + lorenzo_second_order: true, + regression: true, + regression_second_order: true, + prediction_dimension: None, } } None => sz3::CompressionAlgorithm::NoPrediction, }; config = config.compression_algorithm(predictor); - sz3::compress_into_with_config(&data, &config, &mut encoded_bytes).map_err(|err| { + // TODO: avoid extra allocation here + let compressed = sz3::compress_with_config(&data, &config).map_err(|err| { Sz3CodecError::Sz3EncodeFailed { source: Sz3CodingError(err), } })?; + encoded_bytes.extend_from_slice(&compressed); Ok(encoded_bytes) } -/// Decompresses the `encoded` data into an array using SZ3. +/// Decompresses the `encoded` data into an array. /// /// # Errors /// /// Errors with /// - [`Sz3CodecError::HeaderDecodeFailed`] if decoding the header failed -/// - [`Sz3CodecError::Sz3DecodeFailed`] if decoding failed with an opaque error pub fn decompress(encoded: &[u8]) -> Result { - fn decompress_typed( - encoded: &[u8], - shape: &[usize], - ) -> Result, Sz3CodecError> { - if shape.iter().copied().any(|s| s == 0) { - return Ok(Array::from_shape_vec(shape, Vec::new())?); - } - - let (_config, decompressed) = - sz3::decompress(encoded).map_err(|err| Sz3CodecError::Sz3DecodeFailed { - source: Sz3CodingError(err), - })?; - - Ok(Array::from_shape_vec(shape, decompressed.into_data())?) - } - let (header, data) = postcard::take_from_bytes::(encoded).map_err(|err| { Sz3CodecError::HeaderDecodeFailed { @@ -511,162 +605,56 @@ pub fn decompress(encoded: &[u8]) -> Result { } })?; - let decoded = match header.dtype { - Sz3DType::U8 => AnyArray::U8(decompress_typed(data, &header.shape)?), - Sz3DType::I8 => AnyArray::I8(decompress_typed(data, &header.shape)?), - Sz3DType::U16 => AnyArray::U16(decompress_typed(data, &header.shape)?), - Sz3DType::I16 => AnyArray::I16(decompress_typed(data, &header.shape)?), - Sz3DType::U32 => AnyArray::U32(decompress_typed(data, &header.shape)?), - Sz3DType::I32 => AnyArray::I32(decompress_typed(data, &header.shape)?), - Sz3DType::U64 => AnyArray::U64(decompress_typed(data, &header.shape)?), - Sz3DType::I64 => AnyArray::I64(decompress_typed(data, &header.shape)?), - Sz3DType::F32 => AnyArray::F32(decompress_typed(data, &header.shape)?), - Sz3DType::F64 => AnyArray::F64(decompress_typed(data, &header.shape)?), - }; - - Ok(decoded) -} - -/// Decompresses the `encoded` data into a `decoded` array using SZ3. -/// -/// # Errors -/// -/// Errors with -/// - [`Sz3CodecError::HeaderDecodeFailed`] if decoding the header failed -/// - [`Sz3CodecError::MismatchedDecodeIntoArray`] if the `decoded` array is of -/// the wrong dtype or shape -/// - [`Sz3CodecError::Sz3DecodeFailed`] if decoding failed with an opaque error -pub fn decompress_into(encoded: &[u8], decoded: AnyArrayViewMut) -> Result<(), Sz3CodecError> { - fn decompress_into_typed( - encoded: &[u8], - mut decoded: ArrayViewMut, - ) -> Result<(), Sz3CodecError> { - if decoded.is_empty() { - return Ok(()); - } - - let decoded_shape = decoded.shape().to_vec(); - - decoded.with_slice_mut(|mut decoded| { - let decoded_len = decoded.len(); - - let mut builder = sz3::DimensionedData::build_mut(&mut decoded); - - for length in &decoded_shape { - // Sz3 ignores dimensions of length 1 and panics on length zero - // Since they carry no information for Sz3 and we already encode them - // in our custom header, we just skip them here - if *length > 1 { - builder = builder - .dim(*length) - // FIXME: different error code - .map_err(|err| Sz3CodecError::InvalidEncodeShape { - source: Sz3CodingError(err), - shape: decoded_shape.clone(), - })?; - } + let decoded = if header.shape.iter().copied().product::() == 0 { + match header.dtype { + Sz3DType::I32 => { + AnyArray::I32(Array::from_shape_vec(&*header.shape, Vec::new())?.into_dyn()) } - - if decoded_len == 1 { - // If there is only one element, all dimensions will have been skipped, - // so we explicitly encode one dimension of size 1 here - builder = builder - .dim(1) - // FIXME: different error code - .map_err(|err| Sz3CodecError::InvalidEncodeShape { - source: Sz3CodingError(err), - shape: decoded_shape.clone(), - })?; + Sz3DType::I64 => { + AnyArray::I64(Array::from_shape_vec(&*header.shape, Vec::new())?.into_dyn()) } - - let mut decoded = builder - .finish() - // FIXME: different error code - .map_err(|err| Sz3CodecError::InvalidEncodeShape { - source: Sz3CodingError(err), - shape: decoded_shape, - })?; - - sz3::decompress_into_dimensioned(encoded, &mut decoded).map_err(|err| { - Sz3CodecError::Sz3DecodeFailed { - source: Sz3CodingError(err), - } - }) - })?; - - Ok(()) - } - - let (header, data) = - postcard::take_from_bytes::(encoded).map_err(|err| { - Sz3CodecError::HeaderDecodeFailed { - source: Sz3HeaderError(err), + Sz3DType::F32 => { + AnyArray::F32(Array::from_shape_vec(&*header.shape, Vec::new())?.into_dyn()) } - })?; - - if decoded.shape() != &*header.shape { - return Err(Sz3CodecError::MismatchedDecodeIntoArray { - source: AnyArrayAssignError::ShapeMismatch { - src: header.shape.into_owned(), - dst: decoded.shape().to_vec(), - }, - }); - } + Sz3DType::F64 => { + AnyArray::F64(Array::from_shape_vec(&*header.shape, Vec::new())?.into_dyn()) + } + } + } else { + // TODO: avoid extra allocation here + match header.dtype { + Sz3DType::I32 => AnyArray::I32(Array::from_shape_vec( + &*header.shape, + Vec::from(sz3::decompress(data).1.data()), + )?), + Sz3DType::I64 => AnyArray::I64(Array::from_shape_vec( + &*header.shape, + Vec::from(sz3::decompress(data).1.data()), + )?), + Sz3DType::F32 => AnyArray::F32(Array::from_shape_vec( + &*header.shape, + Vec::from(sz3::decompress(data).1.data()), + )?), + Sz3DType::F64 => AnyArray::F64(Array::from_shape_vec( + &*header.shape, + Vec::from(sz3::decompress(data).1.data()), + )?), + } + }; - match (decoded, header.dtype) { - (AnyArrayViewMut::U8(decoded), Sz3DType::U8) => decompress_into_typed(data, decoded), - (AnyArrayViewMut::I8(decoded), Sz3DType::I8) => decompress_into_typed(data, decoded), - (AnyArrayViewMut::U16(decoded), Sz3DType::U16) => decompress_into_typed(data, decoded), - (AnyArrayViewMut::I16(decoded), Sz3DType::I16) => decompress_into_typed(data, decoded), - (AnyArrayViewMut::U32(decoded), Sz3DType::U32) => decompress_into_typed(data, decoded), - (AnyArrayViewMut::I32(decoded), Sz3DType::I32) => decompress_into_typed(data, decoded), - (AnyArrayViewMut::U64(decoded), Sz3DType::U64) => decompress_into_typed(data, decoded), - (AnyArrayViewMut::I64(decoded), Sz3DType::I64) => decompress_into_typed(data, decoded), - (AnyArrayViewMut::F32(decoded), Sz3DType::F32) => decompress_into_typed(data, decoded), - (AnyArrayViewMut::F64(decoded), Sz3DType::F64) => decompress_into_typed(data, decoded), - (decoded, dtype) => Err(Sz3CodecError::MismatchedDecodeIntoArray { - source: AnyArrayAssignError::DTypeMismatch { - src: dtype.into_dtype(), - dst: decoded.dtype(), - }, - }), - } + Ok(decoded) } /// Array element types which can be compressed with SZ3. -pub trait Sz3Element: Copy + sz3::SZ3Compressible + ArrayDType { +pub trait Sz3Element: Copy + sz3::SZ3Compressible { /// The dtype representation of the type const DTYPE: Sz3DType; } -impl Sz3Element for u8 { - const DTYPE: Sz3DType = Sz3DType::U8; -} - -impl Sz3Element for i8 { - const DTYPE: Sz3DType = Sz3DType::I8; -} - -impl Sz3Element for u16 { - const DTYPE: Sz3DType = Sz3DType::U16; -} - -impl Sz3Element for i16 { - const DTYPE: Sz3DType = Sz3DType::I16; -} - -impl Sz3Element for u32 { - const DTYPE: Sz3DType = Sz3DType::U32; -} - impl Sz3Element for i32 { const DTYPE: Sz3DType = Sz3DType::I32; } -impl Sz3Element for u64 { - const DTYPE: Sz3DType = Sz3DType::U64; -} - impl Sz3Element for i64 { const DTYPE: Sz3DType = Sz3DType::I64; } @@ -691,18 +679,6 @@ struct CompressionHeader<'a> { #[derive(Copy, Clone, Debug, Serialize, Deserialize)] #[expect(missing_docs)] pub enum Sz3DType { - #[serde(rename = "u8", alias = "uint8")] - U8, - #[serde(rename = "u16", alias = "uint16")] - U16, - #[serde(rename = "u32", alias = "uint32")] - U32, - #[serde(rename = "u64", alias = "uint64")] - U64, - #[serde(rename = "i8", alias = "int8")] - I8, - #[serde(rename = "i16", alias = "int16")] - I16, #[serde(rename = "i32", alias = "int32")] I32, #[serde(rename = "i64", alias = "int64")] @@ -713,34 +689,9 @@ pub enum Sz3DType { F64, } -impl Sz3DType { - /// Get the corresponding [`AnyArrayDType`] - #[must_use] - pub const fn into_dtype(self) -> AnyArrayDType { - match self { - Self::U8 => AnyArrayDType::U8, - Self::U16 => AnyArrayDType::U16, - Self::U32 => AnyArrayDType::U32, - Self::U64 => AnyArrayDType::U64, - Self::I8 => AnyArrayDType::I8, - Self::I16 => AnyArrayDType::I16, - Self::I32 => AnyArrayDType::I32, - Self::I64 => AnyArrayDType::I64, - Self::F32 => AnyArrayDType::F32, - Self::F64 => AnyArrayDType::F64, - } - } -} - impl fmt::Display for Sz3DType { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { fmt.write_str(match self { - Self::U8 => "u8", - Self::U16 => "u16", - Self::U32 => "u32", - Self::U64 => "u64", - Self::I8 => "i8", - Self::I16 => "i16", Self::I32 => "i32", Self::I64 => "i64", Self::F32 => "f32", @@ -782,12 +733,7 @@ mod tests { )?; let decoded = decompress(&encoded)?; - assert_eq!(decoded, AnyArray::I32(data.clone())); - - let mut decoded = Array::zeros(data.dim()); - decompress_into(&encoded, AnyArrayViewMut::I32(decoded.view_mut()))?; - - assert_eq!(decoded, data); + assert_eq!(decoded, AnyArray::I32(data)); Ok(()) } @@ -812,14 +758,6 @@ mod tests { decoded, AnyArray::F64(Array1::from_vec(data.to_vec()).into_dyn()) ); - - let mut decoded = Array::zeros([data.len()]); - decompress_into( - &encoded, - AnyArrayViewMut::F64(decoded.view_mut().into_dyn()), - )?; - - assert_eq!(decoded, Array1::from_vec(data.to_vec())); } Ok(()) @@ -831,15 +769,21 @@ mod tests { for predictor in [ None, - Some(Sz3Predictor::Interpolation), - Some(Sz3Predictor::InterpolationLorenzo), Some(Sz3Predictor::Regression), + Some(Sz3Predictor::RegressionSecondOrder), + Some(Sz3Predictor::RegressionFirstSecondOrder), Some(Sz3Predictor::LorenzoSecondOrder), + Some(Sz3Predictor::LorenzoSecondOrderRegressionSecondOrder), Some(Sz3Predictor::LorenzoSecondOrderRegression), + Some(Sz3Predictor::LorenzoSecondOrderRegressionFirstSecondOrder), Some(Sz3Predictor::Lorenzo), + Some(Sz3Predictor::LorenzoRegressionSecondOrder), Some(Sz3Predictor::LorenzoRegression), + Some(Sz3Predictor::LorenzoRegressionFirstSecondOrder), Some(Sz3Predictor::LorenzoFirstSecondOrder), + Some(Sz3Predictor::LorenzoFirstSecondOrderRegressionSecondOrder), Some(Sz3Predictor::LorenzoFirstSecondOrderRegression), + Some(Sz3Predictor::LorenzoFirstSecondOrderRegressionFirstSecondOrder), ] { let encoded = compress( data.view(), @@ -847,47 +791,8 @@ mod tests { &Sz3ErrorBound::Absolute { abs: 0.1 }, )?; let _decoded = decompress(&encoded)?; - - let mut decoded = Array::zeros(data.dim()); - decompress_into( - &encoded, - AnyArrayViewMut::F64(decoded.view_mut().into_dyn()), - )?; - } - - Ok(()) - } - - #[test] - fn all_dtypes() -> Result<(), Sz3CodecError> { - fn compress_decompress( - iter: impl Clone + IntoIterator, - view_mut: impl for<'a> Fn(ArrayViewMut<'a, T, IxDyn>) -> AnyArrayViewMut<'a>, - ) -> Result<(), Sz3CodecError> { - let encoded = compress( - Array::from_iter(iter.clone()).view(), - default_predictor().as_ref(), - &Sz3ErrorBound::Absolute { abs: 2.0 }, - )?; - let _decoded = decompress(&encoded)?; - - let mut decoded = Array::::zeros([iter.into_iter().count()]).into_dyn(); - decompress_into(&encoded, view_mut(decoded.view_mut().into_dyn()))?; - - Ok(()) } - compress_decompress(0_u8..42, |x| AnyArrayViewMut::U8(x))?; - compress_decompress(-42_i8..42, |x| AnyArrayViewMut::I8(x))?; - compress_decompress(0_u16..42, |x| AnyArrayViewMut::U16(x))?; - compress_decompress(-42_i16..42, |x| AnyArrayViewMut::I16(x))?; - compress_decompress(0_u32..42, |x| AnyArrayViewMut::U32(x))?; - compress_decompress(-42_i32..42, |x| AnyArrayViewMut::I32(x))?; - compress_decompress(0_u64..42, |x| AnyArrayViewMut::U64(x))?; - compress_decompress(-42_i64..42, |x| AnyArrayViewMut::I64(x))?; - compress_decompress((-42_i16..42).map(f32::from), |x| AnyArrayViewMut::F32(x))?; - compress_decompress((-42_i16..42).map(f64::from), |x| AnyArrayViewMut::F64(x))?; - Ok(()) } } diff --git a/codecs/sz3/tests/config.rs b/codecs/sz3/tests/config.rs index aa5bda0f2..8d22c628f 100644 --- a/codecs/sz3/tests/config.rs +++ b/codecs/sz3/tests/config.rs @@ -1,9 +1,6 @@ #![expect(missing_docs)] -use ::{ - ndarray as _, num_traits as _, postcard as _, schemars as _, sz3 as _, thiserror as _, - zstd_sys as _, -}; +use ::{ndarray as _, postcard as _, schemars as _, sz3 as _, thiserror as _, zstd_sys as _}; use numcodecs::StaticCodec; use numcodecs_sz3::Sz3Codec; @@ -62,7 +59,7 @@ fn config_predictor() { Deserialize::deserialize(json!({ "eb_mode": "rel", "eb_rel": 1.0, - "predictor": "interpolation", + "predictor": "linear-interpolation", })) .unwrap(), ); @@ -71,7 +68,7 @@ fn config_predictor() { Deserialize::deserialize(json!({ "eb_mode": "rel", "eb_rel": 1.0, - "predictor": "interpolation-lorenzo", + "predictor": "cubic-interpolation-lorenzo", })) .unwrap(), ); diff --git a/codecs/sz3/tests/schema.json b/codecs/sz3/tests/schema.json index 0c5fb52db..c32d21c62 100644 --- a/codecs/sz3/tests/schema.json +++ b/codecs/sz3/tests/schema.json @@ -7,48 +7,98 @@ "oneOf": [ { "type": "string", - "const": "interpolation", - "description": "Interpolation" + "const": "linear-interpolation", + "description": "Linear interpolation" }, { "type": "string", - "const": "interpolation-lorenzo", - "description": "Interpolation + Lorenzo predictor" + "const": "cubic-interpolation", + "description": "Cubic interpolation" + }, + { + "type": "string", + "const": "linear-interpolation-lorenzo", + "description": "Linear interpolation + Lorenzo predictor" + }, + { + "type": "string", + "const": "cubic-interpolation-lorenzo", + "description": "Cubic interpolation + Lorenzo predictor" }, { "type": "string", "const": "regression", "description": "1st order regression" }, + { + "type": "string", + "const": "regression2", + "description": "2nd order regression" + }, + { + "type": "string", + "const": "regression-regression2", + "description": "1st+2nd order regression" + }, { "type": "string", "const": "lorenzo2", "description": "2nd order Lorenzo predictor" }, + { + "type": "string", + "const": "lorenzo2-regression2", + "description": "2nd order Lorenzo predictor + 2nd order regression" + }, { "type": "string", "const": "lorenzo2-regression", "description": "2nd order Lorenzo predictor + 1st order regression" }, + { + "type": "string", + "const": "lorenzo2-regression-regression2", + "description": "2nd order Lorenzo predictor + 1st order regression" + }, { "type": "string", "const": "lorenzo", "description": "1st order Lorenzo predictor" }, + { + "type": "string", + "const": "lorenzo-regression2", + "description": "1st order Lorenzo predictor + 2nd order regression" + }, { "type": "string", "const": "lorenzo-regression", "description": "1st order Lorenzo predictor + 1st order regression" }, + { + "type": "string", + "const": "lorenzo-regression-regression2", + "description": "1st order Lorenzo predictor + 1st and 2nd order regression" + }, { "type": "string", "const": "lorenzo-lorenzo2", "description": "1st+2nd order Lorenzo predictor" }, + { + "type": "string", + "const": "lorenzo-lorenzo2-regression2", + "description": "1st+2nd order Lorenzo predictor + 2nd order regression" + }, { "type": "string", "const": "lorenzo-lorenzo2-regression", "description": "1st+2nd order Lorenzo predictor + 1st order regression" + }, + { + "type": "string", + "const": "lorenzo-lorenzo2-regression-regression2", + "description": "1st+2nd order Lorenzo predictor + 1st+2nd order regression" } ], "description": "SZ3 predictor" @@ -58,13 +108,13 @@ } ], "description": "Predictor", - "default": "interpolation-lorenzo" + "default": "cubic-interpolation-lorenzo" }, "_version": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", "description": "The codec's encoding format version. Do not provide this parameter explicitly.", - "default": "0.2.0" + "default": "0.1.0" } }, "unevaluatedProperties": false, diff --git a/codecs/sz3/tests/schema.rs b/codecs/sz3/tests/schema.rs index 9e40080e0..cb3213aba 100644 --- a/codecs/sz3/tests/schema.rs +++ b/codecs/sz3/tests/schema.rs @@ -1,8 +1,8 @@ #![expect(missing_docs)] use ::{ - ndarray as _, num_traits as _, postcard as _, schemars as _, serde as _, serde_json as _, - sz3 as _, thiserror as _, zstd_sys as _, + ndarray as _, postcard as _, schemars as _, serde as _, serde_json as _, sz3 as _, + thiserror as _, zstd_sys as _, }; use numcodecs::{DynCodecType, StaticCodecType};