Issue #963 dump meta swap model nc

docs/api/changelog.rst

@@ -106,6 +106,9 @@ Added
 - Functionality to dump and load MODFLOW 6 simulations to/from zarr and zipstore
   formats. See :meth:`imod.mf6.Modflow6Simulation.dump` and
   :meth:`imod.mf6.Modflow6Simulation.from_file` for more information.
+- Functionality to dump and load MetaSwap models to/from netcdf
+  format. See :meth:`imod.msw.MetaSwapModel.dump` and
+  :meth:`imod.msw.MetaSwapModel.from_file` for more information.
 
 Changed
 ~~~~~~~

docs/api/msw.rst

@@ -11,6 +11,7 @@ Model objects & methods
 
     MetaSwapModel
     MetaSwapModel.write
+    MetaSwapModel.dump
     MetaSwapModel.from_imod5_data
     MetaSwapModel.regrid_like
     MetaSwapModel.clip_box
@@ -172,4 +173,4 @@ Mappings
     CouplerMapping.clip_box
     CouplerMapping.from_imod5_data
     CouplerMapping.get_regrid_methods
-    CouplerMapping.write
+    CouplerMapping.write

imod/common/utilities/dump_model.py

@@ -0,0 +1,87 @@
+import collections
+from pathlib import Path
+from typing import Any, Optional
+
+import tomli_w
+
+from imod.common.interfaces.idict import IDict
+from imod.common.serializer import EngineType
+from imod.logging.logging_decorators import standard_log_decorator
+from imod.mf6.validation_settings import ValidationSettings
+from imod.schemata import ValidationError
+
+
+@standard_log_decorator()
+def dump_model(
+    model: IDict,
+    directory,
+    modelname,
+    validate: Optional[bool] = True,
+    mdal_compliant: bool = False,
+    crs: Optional[Any] = None,
+    engine: EngineType = "netcdf4",
+) -> Path:
+    """
+    Dump  model to files. Writes a model definition as .TOML file, which
+    points to data for each package. Each package is stored as a separate
+    NetCDF. Structured grids are saved as regular NetCDFs, unstructured
+    grids are saved as UGRID NetCDF. Structured grids are always made GDAL
+    compliant, unstructured grids can be made MDAL compliant optionally.
+
+    Parameters
+    ----------
+    directory: str or Path
+        directory to dump simulation into.
+    modelname: str
+        modelname, will be used to create a subdirectory.
+    validate: bool, optional
+        Whether to validate simulation data. Defaults to True.
+    mdal_compliant: bool, optional
+        Convert data with
+        :func:`imod.prepare.spatial.mdal_compliant_ugrid2d` to MDAL
+        compliant unstructured grids. Defaults to False.
+    crs: Any, optional
+        Anything accepted by rasterio.crs.CRS.from_user_input
+        Requires ``rioxarray`` installed.
+    engine : str, optional
+        File engine used to write packages. Options are ``'netcdf4'``,
+        ``'zarr'``, and ``'zarr.zip'``. NetCDF4 is readable by many other
+        softwares, for example QGIS. Zarr is optimized for big data, cloud
+        storage and parallel access. The ``'zarr.zip'`` option is an
+        experimental option which creates a zipped zarr store in a single
+        file, which is easier to copy and automatically compresses data as
+        well. Default is ``'netcdf4'``.
+
+    """
+    modeldirectory = Path(directory) / modelname
+    modeldirectory.mkdir(exist_ok=True, parents=True)
+
+    # validation currently only supports MF6, but we want to keep the option to turn it on for other
+    if hasattr(model, "validate") and callable(getattr(model, "validate")):
+        validation_context = ValidationSettings(validate=validate)
+        if validation_context.validate:
+            statusinfo = model.validate(modelname, validation_context)
+            if statusinfo.has_errors():
+                raise ValidationError(statusinfo.to_string())
+
+    toml_content: dict = collections.defaultdict(dict)
+
+    for pkgname, pkg in model.items():
+        pkg_path = pkg.to_file(
+            modeldirectory,
+            pkgname,
+            mdal_compliant=mdal_compliant,
+            crs=crs,
+            engine=engine,
+        )
+        toml_content[type(pkg).__name__][pkgname] = pkg_path.name
+
+    # simulation settings are only relevant/present for MetaSwap models (msw)
+    if hasattr(model, "simulation_settings"):
+        toml_content["simulation_settings"] = model.simulation_settings
+
+    toml_path = modeldirectory / f"{modelname}.toml"
+    with open(toml_path, "wb") as f:
+        tomli_w.dump(toml_content, f)
+
+    return toml_path

imod/common/utilities/value_filters.py

@@ -1,3 +1,4 @@
+import numbers
 from typing import Any
 
 import numpy as np
@@ -7,6 +8,17 @@
 from imod.typing import GridDataArray, GridDataset
 
 
+def is_scalar_nan(da: GridDataArray):
+    """
+    Test if is_scalar_nan, carefully avoid loading grids in memory
+    """
+    scalar_data: bool = is_scalar(da)
+    if scalar_data:
+        stripped_value = da.to_numpy()[()]
+        return isinstance(stripped_value, numbers.Real) and np.isnan(stripped_value)  # type: ignore[call-overload]
+    return False
+
+
 def is_valid(value: Any) -> bool:
     """
     Filters values that are None, False, or a numpy.bool_ False.

imod/mf6/model.py

@@ -12,7 +12,6 @@
 import jinja2
 import numpy as np
 import tomli
-import tomli_w
 import xarray as xr
 import xugrid as xu
 from jinja2 import Template
@@ -22,6 +21,7 @@
 from imod.common.serializer import EngineType
 from imod.common.statusinfo import NestedStatusInfo, StatusInfo, StatusInfoBase
 from imod.common.utilities.clip import clip_box_dataset
+from imod.common.utilities.dump_model import dump_model
 from imod.common.utilities.mask import mask_all_packages
 from imod.common.utilities.regrid import _regrid_like
 from imod.common.utilities.schemata import (
@@ -647,30 +647,27 @@ def dump(
             file, which is easier to copy and automatically compresses data as
             well. Default is ``'netcdf4'``.
 
-        """
-        modeldirectory = pathlib.Path(directory) / modelname
-        modeldirectory.mkdir(exist_ok=True, parents=True)
-        validation_context = ValidationSettings(validate=validate)
-        if validation_context.validate:
-            statusinfo = self.validate(modelname, validation_context)
-            if statusinfo.has_errors():
-                raise ValidationError(statusinfo.to_string())
-
-        toml_content: dict[str, Any] = collections.defaultdict(dict)
+        Returns
+        -------
+        Path
+            Path to the created toml file which contains the paths to the dumped package files. The package files are dumped in the same directory as the toml file.
 
-        for pkgname, pkg in self.items():
-            pkg_path = pkg.to_file(
-                modeldirectory,
-                pkgname,
-                mdal_compliant=mdal_compliant,
-                crs=crs,
-                engine=engine,
-            )
-            toml_content[type(pkg).__name__][pkgname] = pkg_path.name
+        Example
+        -------
+        >>> tmp_path = tmpdir_factory.mktemp(name)
+        >>> toml_path = mf6_model.dump(tmp_path, name, engine=engine, validate=False)
+        >>> back = ModflowModel.from_file(tmp_path, name)
+        """
 
-        toml_path = modeldirectory / f"{modelname}.toml"
-        with open(toml_path, "wb") as f:
-            tomli_w.dump(toml_content, f)
+        toml_path = dump_model(
+            self,
+            directory,
+            modelname,
+            validate=validate,
+            mdal_compliant=mdal_compliant,
+            crs=crs,
+            engine=engine,
+        )
 
         return toml_path
 

imod/mf6/pkgbase.py

@@ -1,16 +1,14 @@
 import abc
-import numbers
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Mapping, Optional, Self, final
 
-import numpy as np
 import xarray as xr
 import xugrid as xu
-from xarray.core.utils import is_scalar
 
 import imod
 from imod.common.interfaces.ipackagebase import IPackageBase
 from imod.common.serializer import EngineType, create_package_serializer
+from imod.common.utilities.value_filters import is_scalar_nan
 from imod.typing.grid import (
     GridDataArray,
     GridDataset,
@@ -32,17 +30,6 @@
 UTIL_PACKAGES = ("ats", "hpc")
 
 
-def _is_scalar_nan(da: GridDataArray):
-    """
-    Test if is_scalar_nan, carefully avoid loading grids in memory
-    """
-    scalar_data: bool = is_scalar(da)
-    if scalar_data:
-        stripped_value = da.to_numpy()[()]
-        return isinstance(stripped_value, numbers.Real) and np.isnan(stripped_value)  # type: ignore[call-overload]
-    return False
-
-
 class PackageBase(IPackageBase, abc.ABC):
     """
     This class is used for storing a collection of Xarray DataArrays or UgridDataArrays
@@ -148,7 +135,7 @@ def from_file(cls, path: str | Path, **kwargs) -> Self:
 
         # Replace NaNs by None
         for key, value in dataset.items():
-            if _is_scalar_nan(value):
+            if is_scalar_nan(value):
                 dataset[key] = None
 
         # to_netcdf converts strings into NetCDF "variable‑length UTF‑8 strings"

imod/msw/model.py

@@ -1,4 +1,5 @@
 import collections
+import inspect
 import warnings
 from copy import copy, deepcopy
 from datetime import datetime
@@ -8,10 +9,15 @@
 import cftime
 import jinja2
 import numpy as np
+import tomli
 import xarray as xr
 
+import imod.msw
 from imod.common.constants import MaskValues
+from imod.common.interfaces.idict import IDict
+from imod.common.serializer import EngineType
 from imod.common.utilities.clip import clip_by_grid
+from imod.common.utilities.dump_model import dump_model
 from imod.common.utilities.partitioninfo import create_partition_info
 from imod.common.utilities.regrid import regrid_imod5_cap_data
 from imod.common.utilities.version import prepend_content_with_version_info
@@ -108,7 +114,7 @@
             self[k] = v
 
 
-class MetaSwapModel(Model):
+class MetaSwapModel(Model, IDict):
     """
     Contains data and writes consistent model input files
 
@@ -369,6 +375,90 @@
         for pkgname in self:
             self[pkgname].write(directory, index, svat, mf6_dis, mf6_wel)
 
+    @classmethod
+    def from_file(cls, directory, modelname):
+        pkg_classes = {
+            name: pkg_cls
+            for name, pkg_cls in inspect.getmembers(imod.msw, inspect.isclass)
+            if issubclass(pkg_cls, MetaSwapPackage)
+        }
+        modeldirectory = Path(directory) / modelname
+        toml_path = modeldirectory / f"{modelname}.toml"
+        with open(toml_path, "rb") as f:
+            toml_content = tomli.load(f)
+
+        parentdir = toml_path.parent
+        simulation_settings = toml_content.get("simulation_settings", {})
+        unsa_svat_path = simulation_settings.get("unsa_svat_path", "")
+        instance = cls(unsa_svat_path, simulation_settings)
+
+        for key, entry in toml_content.items():
+            if key != "simulation_settings":
+                for pkgname, path in entry.items():
+                    pkg_cls = pkg_classes[key]
+                    instance[pkgname] = pkg_cls.from_file(parentdir / path)
+
+        return instance
+
+    def dump(
+        self,
+        directory: Union[str, Path],
+        modelname: Optional[str] = None,
+        validate: Optional[bool] = True,
+        mdal_compliant: bool = False,
+        crs: Optional[str] = None,
+        engine: EngineType = "netcdf4",
+    ):
+        """
+        Dump model packages to netCDF files and create a toml file with paths to these files.
+        The MetaSWAP model can be reloaded from the dumped files using the :func:`from_file` method.
+
+        Parameters
+        ----------
+        directory: Path or str
+            Directory to dump model in. A subdirectory with the name of the model will be created in this directory, and the files will be dumped there.
+        modelname: str, optional
+            Name of the model. This will be used as the name of the subdirectory where the files are dumped, and in the name of the toml file. If not provided, it defaults to the value of ``self._model_name``.
+        validate: bool, optional
+            Whether to perform validation before dumping the model. If True, the model will be validated using the validation functionality in iMOD. If validation errors are found, a ValidationError is raised and the model is not dumped. Default is True.
+        mdal_compliant: bool, optional
+            Whether to write the files in a format compliant with the MDAL specification. This can be used to make the files compatible with software that supports MDAL, such as QGIS. Default is False.
+        crs: str, optional
+            Coordinate reference system to use in the dumped files. This should be a string in a format recognized by the pyproj library, for example "EPSG:28992". If not provided, no CRS information is included in the files.
+        engine: EngineType, optional
+            File engine used to write packages.
+                    engine : str, optional
+            File engine used to write packages. Options are ``'netcdf4'``,
+            ``'zarr'``, and ``'zarr.zip'``. NetCDF4 is readable by many other
+            softwares, for example QGIS. Zarr is optimized for big data, cloud
+            storage and parallel access. The ``'zarr.zip'`` option is an
+            experimental option which creates a zipped zarr store in a single
+            file, which is easier to copy and automatically compresses data as
+            well. Default is ``'netcdf4'``.
+
+        Returns
+        -------
+        Path
+            Path to the created toml file which contains the paths to the dumped package files. The package files are dumped in the same directory as the toml file.
+
+        Example
+        -------
+        >>> tmp_path = tmpdir_factory.mktemp(name)
+        >>> toml_path = msw_model.dump(tmp_path, name, engine=engine, validate=False)
+        >>> back = MetaSwapModel.from_file(tmp_path, name)
+        """
+
+        toml_path = dump_model(
+            self,
+            directory=directory,
+            modelname=modelname,
+            validate=validate,
+            mdal_compliant=mdal_compliant,
+            crs=crs,
+            engine=engine,
+        )
+        return toml_path
+
     def regrid_like(
         self,
         mf6_regridded_dis: StructuredDiscretization,
@@ -679,3 +769,6 @@
         model["time_oc"] = TimeOutputControl(times_da)
 
         return model
+
+
+# make a read function for packages from netcdf