Introduction of returns to our system

packages/scratch-core/pyproject.toml

@@ -10,11 +10,13 @@ authors = [
 readme = "README.md"
 
 dependencies = [
+  "loguru>=0.7.3",
   "numpy>=2.3.4",
   "pillow>=12.0.0",
-  "scikit-image>=0.25.2",
+  "pydantic>=2.12.4",
+  "returns>=0.26.0",
+  "scipy>=1.16.3",
   "surfalize~=0.16.6",
-  "numpydantic>=1.7.0",
   "x3p @ git+https://github.com/giacomomarchioro/pyx3p.git#81c0f764cf321e56dc41e9e3c71d14e97d5bc3ae",
 ]
 

packages/scratch-core/src/container_models/__init__.py

@@ -0,0 +1,15 @@
+"""
+Immutable data container models for railway-oriented programming pipelines.
+
+This module provides Pydantic-based data models that are propagated through railway
+functions in functional pipelines. These models serve as type-safe, validated containers
+for scientific and imaging data, ensuring data integrity as it flows through processing
+pipelines.
+
+Notes
+-----
+These models are designed specifically for railway-oriented programming where data
+flows through a sequence of transformations. The immutability ensures that each
+function in the pipeline receives unmodified input, preventing side effects and
+making pipelines easier to reason about and debug.
+"""

packages/scratch-core/src/container_models/base.py

@@ -0,0 +1,59 @@
+from collections.abc import Sequence
+from functools import partial
+from typing import Annotated, TypeAlias
+
+from numpy import array, bool_, float64, number, uint8
+from numpy.typing import DTypeLike, NDArray
+from pydantic import BaseModel, BeforeValidator, ConfigDict, PlainSerializer
+
+
+def serialize_ndarray[T: number](array_: NDArray[T]) -> list[T]:
+    """Serialize numpy array to a Python list for JSON serialization."""
+    return array_.tolist()
+
+
+def coerce_to_array[T: number](
+    dtype: DTypeLike, value: Sequence[T] | NDArray[T] | None
+) -> NDArray[T] | None:
+    """
+    Coerce input to dtype numpy array.
+
+    Handles JSON deserialization where Python creates int64 integers by default.
+    """
+    if isinstance(value, Sequence):
+        try:
+            return array(value, dtype=dtype)
+        except OverflowError as ofe:
+            raise ValueError("Array's value(s) out of range") from ofe
+
+    return value
+
+
+ScanMapRGBA: TypeAlias = Annotated[
+    NDArray[uint8],
+    BeforeValidator(partial(coerce_to_array, uint8)),
+    PlainSerializer(serialize_ndarray),
+]
+
+ScanMap2DArray = ScanVectorField2DArray = UnitVector3DArray = Annotated[
+    NDArray[float64],
+    BeforeValidator(partial(coerce_to_array, float64)),
+    PlainSerializer(serialize_ndarray),
+]
+
+MaskArray = Annotated[
+    NDArray[bool_],
+    BeforeValidator(partial(coerce_to_array, bool_)),
+    PlainSerializer(serialize_ndarray),
+]
+
+
+class ConfigBaseModel(BaseModel):
+    """Base model with common configuration for all pydantic models in this project."""
+
+    model_config = ConfigDict(
+        frozen=True,
+        extra="forbid",
+        arbitrary_types_allowed=True,
+        regex_engine="rust-regex",
+    )

packages/scratch-core/src/container_models/light_source.py

@@ -0,0 +1,48 @@
+from functools import cached_property
+import numpy as np
+from pydantic import Field
+from .base import UnitVector3DArray, ConfigBaseModel
+
+
+class LightSource(ConfigBaseModel):
+    """
+    Representation of a light source using an angular direction (azimuth and elevation)
+    together with a derived 3D unit direction vector.
+    """
+
+    azimuth: float = Field(
+        ...,
+        description="Horizontal angle in degrees measured from the –x axis in the x–y plane. "
+        "0° is –x direction, 90° is +y direction, 180° is +x direction.",
+        examples=[90, 45, 180],
+        ge=0,
+        le=360,
+    )
+    elevation: float = Field(
+        ...,
+        description="Vertical angle in degrees measured from the x–y plane. "
+        "0° is horizontal, +90° is upward (+z), –90° is downward (–z).",
+        examples=[90, 45, 180],
+        ge=-90,
+        le=90,
+    )
+
+    @cached_property
+    def unit_vector(self) -> UnitVector3DArray:
+        """
+        Returns the unit direction vector [x, y, z] corresponding to the azimuth and
+        elevation angles. The conversion follows a spherical-coordinate convention:
+        azimuth defines the horizontal direction, and elevation defines the vertical
+        tilt relative to the x–y plane.
+        """
+        azimuth = np.deg2rad(self.azimuth)
+        elevation = np.deg2rad(self.elevation)
+        vec = np.array(
+            [
+                -np.cos(azimuth) * np.cos(elevation),
+                np.sin(azimuth) * np.cos(elevation),
+                np.sin(elevation),
+            ]
+        )
+        vec.setflags(write=False)
+        return vec

packages/scratch-core/src/container_models/scan_image.py

@@ -0,0 +1,26 @@
+from pydantic import Field
+from .base import ScanMap2DArray, ConfigBaseModel
+
+
+class ScanImage(ConfigBaseModel):
+    """
+    A 2D image/array of floats.
+
+    Used for: depth maps, intensity maps, single-channel images.
+    Shape: (height, width)
+    """
+
+    data: ScanMap2DArray
+    scale_x: float = Field(..., gt=0.0, description="pixel size in meters (m)")
+    scale_y: float = Field(..., gt=0.0, description="pixel size in meters (m)")
+    meta_data: dict = Field(default_factory=dict)
+
+    @property
+    def width(self) -> int:
+        """The image width in pixels."""
+        return self.data.shape[1]
+
+    @property
+    def height(self) -> int:
+        """The image height in pixels."""
+        return self.data.shape[0]

packages/scratch-core/src/conversion/__init__.py

@@ -1,3 +1,33 @@
-from conversion.subsample import subsample_array
+"""
+Staging area for MATLAB-to-Python converted code.
 
-__all__ = ("subsample_array",)
+This module serves as a temporary dumping ground for newly translated MATLAB code
+before it undergoes full integration into the main codebase. Code placed here is
+in a transitional state and may not yet conform to project standards, architectural
+patterns, or best practices.
+
+Purpose
+-------
+The conversion module provides a designated space where developers can:
+1. Place initial MATLAB-to-Python translations without disrupting the main codebase
+2. Test and validate converted algorithms in isolation
+3. Iteratively refactor and improve code quality before final integration
+4. Document conversion notes, gotchas, and MATLAB-specific behaviors
+
+Workflow
+--------
+1. **Convert**: Translate MATLAB code to Python and place it in this module
+2. **Validate**: Verify the converted code produces correct results
+3. **Refactor**: Adapt code to project standards (type hints, Pydantic models, etc.)
+4. **Integrate**: Move refined code to appropriate modules (pipelines, preprocessors, etc.)
+5. **Remove**: Delete the staging code once integration is complete
+
+After migration, the staging code in this module should be deleted.
+
+Warnings
+--------
+- DO NOT import from this module in production code
+- DO NOT depend on code in this module for long-term functionality
+- Code here may be incomplete, buggy, or subject to breaking changes
+- This module should remain empty or nearly empty in a mature codebase
+"""

packages/scratch-core/src/conversion/gaussian_filter.py

@@ -5,7 +5,7 @@
 from scipy import ndimage
 from scipy.special import lambertw
 
-from utils.array_definitions import ScanMap2DArray
+from container_models.base import ScanMap2DArray
 
 
 @cache

packages/scratch-core/src/conversion/leveling/core.py

@@ -6,7 +6,7 @@
     compute_root_mean_square,
 )
 from conversion.leveling.solver.utils import compute_image_center
-from image_generation.data_formats import ScanImage
+from container_models.scan_image import ScanImage
 
 
 def level_map(

packages/scratch-core/src/conversion/leveling/solver/grid.py

@@ -1,6 +1,6 @@
 import numpy as np
 from numpy.typing import NDArray
-from image_generation.data_formats import ScanImage
+from container_models.scan_image import ScanImage
 
 
 def get_2d_grid(

packages/scratch-core/src/conversion/leveling/solver/utils.py

@@ -2,7 +2,7 @@
 from typing import Any
 from numpy.typing import NDArray
 
-from image_generation.data_formats import ScanImage
+from container_models.scan_image import ScanImage
 
 
 def compute_root_mean_square(data: NDArray[Any]) -> float:

packages/scratch-core/src/conversion/mask.py

@@ -1,6 +1,6 @@
 import numpy as np
 
-from utils.array_definitions import ScanMap2DArray, MaskArray
+from container_models.base import ScanMap2DArray, MaskArray
 
 
 def mask_2d_array(

packages/scratch-core/src/conversion/resample.py

@@ -1,11 +1,11 @@
 from typing import Optional
 
 import numpy as np
-from numpydantic import NDArray
+from numpy.typing import NDArray
 from scipy import ndimage
 
-from image_generation.data_formats import ScanImage
-from utils.array_definitions import MaskArray
+from container_models.scan_image import ScanImage
+from container_models.base import MaskArray
 
 
 def resample_image_and_mask(