Merge pull request #163 from SasView/full-hdf-save

Full hdf save

Author: rprospero

sasdata/data.py

@@ -2,6 +2,7 @@
 
 import h5py
 import numpy as np
+from h5py._hl.group import Group as HDF5Group
 
 from sasdata import dataset_types
 from sasdata.dataset_types import DatasetType
@@ -21,7 +22,7 @@ def __init__(
         self.name = name
         # validate data contents
         if not all([key in dataset_type.optional or key in dataset_type.required for key in data_contents]):
-            raise ValueError("Columns don't match the dataset type")
+            raise ValueError(f"Columns don't match the dataset type: {[key for key in data_contents]}")
         self._data_contents = data_contents
         self._verbose = verbose
 
@@ -103,13 +104,27 @@ def from_json(obj):
             metadata=Metadata.from_json(obj["metadata"]),
         )
 
-    def save_h5(self, path: str | typing.BinaryIO):
+    def _save_h5(self, sasentry: HDF5Group):
         """Export data into HDF5 file"""
+        sasentry.attrs["name"] = self.name
+        self.metadata.as_h5(sasentry)
+
+        # We export each data set into its own entry, so we only ever
+        # need sasdata01
+        group = sasentry.create_group("sasdata01")
+        for idx, (key, sasdata) in enumerate(self._data_contents.items()):
+            sasdata.as_h5(group, key)
+
+
+    @staticmethod
+    def save_h5(data: dict[str, typing.Self], path: str | typing.BinaryIO):
         with h5py.File(path, "w") as f:
-            f.attrs["name"] = self.name
-            for idx, (key, entry) in enumerate(self._data_contents.items()):
-                group = f.create_group(f"sasentry{idx:02d}")
-                self.metadata.as_h5(group)
+            for idx, (key, data) in enumerate(data.items()):
+                sasentry = f.create_group(f"sasentry{idx+1:02d}")
+                if not key.startswith("sasentry"):
+                    sasentry.attrs["sasview_key"] = key
+                data._save_h5(sasentry)
+
 
 
 class SasDataEncoder(MetadataEncoder):
@@ -125,7 +140,7 @@ def default(self, obj):
             case SasData():
                 return {
                     "name": obj.name,
-                    "data_contents": {},
+                    "data_contents": obj._data_contents,
                     "type": obj.dataset_type,
                     "mask": obj.mask,
                     "metadata": obj.metadata,

sasdata/dataset_types.py

@@ -27,7 +27,7 @@ class DatasetType:
 two_dim = DatasetType(
             name="2D I vs Q",
             required=["Qx", "Qy", "I"],
-            optional=["dQx", "dQy", "dI", "Qz", "ShadowFactor"],
+            optional=["dQx", "dQy", "dI", "Qz", "ShadowFactor", "mask"],
             expected_orders=[
                 ["Qx", "Qy", "I"],
                 ["Qx", "Qy", "I", "dI"],

sasdata/metadata.py

@@ -71,7 +71,7 @@ class Rot3:
     def from_json(obj: dict) -> Quantity | None:
         if obj is None:
             return None
-        return Vec3(
+        return Rot3(
             roll=from_json_quantity(obj["roll"]),
             pitch=from_json_quantity(obj["pitch"]),
             yaw=from_json_quantity(obj["yaw"]),
@@ -165,7 +165,7 @@ def from_json(obj):
             size=Vec3.from_json(obj["size"]),
             size_name=obj["size_name"],
             name=obj["name"],
-            type_=obj["type_"],
+            type_=obj["type"],
         )
 
 
@@ -541,11 +541,11 @@ class Metadata:
     raw: MetaNode
 
     def summary(self):
-        run_string = self.run[0] if len(self.run) == 1 else self.run
+        run_string = str(self.run[0] if len(self.run) == 1 else self.run)
         return (
             f"  {self.title}, Run: {run_string}\n"
             + "  "
-            + "=" * len(self.title if self.title else "")
+            + "=" * len(str(self.title))
             + "======="
             + "=" * len(run_string)
             + "\n\n"
@@ -558,12 +558,12 @@ def summary(self):
     @staticmethod
     def from_json(obj):
         return Metadata(
-            title=obj["title"],
+            title=obj["title"] if obj["title"] else None,
             run=obj["run"],
-            definition=obj["definition"],
+            definition=obj["definition"] if obj["definition"] else None,
             process=[Process.from_json(p) for p in obj["process"]],
-            sample=Sample.from_json(obj["sample"]),
-            instrument=Instrument.from_json(obj["instrument"]),
+            sample=Sample.from_json(obj["sample"]) if obj["sample"] else None,
+            instrument=Instrument.from_json(obj["instrument"]) if obj["instrument"] else None,
             raw=MetaNode.from_json(obj["raw"]),
         )
 
@@ -591,10 +591,12 @@ def default(self, obj):
         match obj:
             case None:
                 return None
+            case bytes():
+                return obj.decode("utf-8")
             case NamedUnit():
                 return obj.name
             case Quantity():
-                return {"value": obj.value, "units": obj.units}
+                return {"value": obj.value, "units": obj.units.ascii_symbol}
             case ndarray():
                 return {
                     "type": "ndarray",

sasdata/quantities/quantity.py

@@ -1401,8 +1401,9 @@ def string_repr(self):
 
     def as_h5(self, group: h5py.Group, name: str):
         """Add this data onto a group as a dataset under the given name"""
-        data = group.create_dataset(name, data=[self.value])
-        data.attrs["units"] = self.units.symbol
+        boxed = self.value if type(self.value) is np.ndarray else [self.value]
+        data = group.create_dataset(name, data=boxed)
+        data.attrs["units"] = self.units.ascii_symbol
 
 
 class NamedQuantity[QuantityType](Quantity[QuantityType]):

sasdata/quantities/unit_parser.py

@@ -19,7 +19,7 @@ def split_unit_str(unit_str: str) -> list[str]:
 def validate_unit_str(unit_str: str) -> bool:
     """Validate whether unit_str is valid. This doesn't mean that the unit specified in unit_str exists but rather it
     only consists of letters, and numbers as a unit string should."""
-    return fullmatch(r"[A-Za-zΩµ%Å^1-9\-\+/\ \._]+", unit_str) is not None
+    return fullmatch(r"[A-Za-zΩµ%Å^1-9⁻¹-⁹\-\+/\ \._]+", unit_str) is not None
 
 
 def parse_single_unit(
@@ -127,7 +127,7 @@ def parse_unit(unit_str: str, longest_unit: bool = True) -> Unit:
         return result
     try:
         if not validate_unit_str(unit_str):
-            raise ValueError("unit_str contains forbidden characters.")
+            raise ValueError(f"unit_str ({unit_str}) contains forbidden characters.")
         parsed_unit = Unit(1, Dimensions())
         unit_stack = parse_unit_stack(unit_str, longest_unit)
         for unit in unit_stack:
@@ -190,6 +190,8 @@ def parse_named_unit_from_group(unit_str: str, from_group: UnitGroup) -> NamedUn
 
 
 def parse(string: str, name_lookup: bool = True, longest_unit: bool = True, lookup_rtol: float = 1e-14):
+    if type(string) is not str:
+        string = string.decode("utf-8")
     unit = parse_unit(string, longest_unit=longest_unit)
     if name_lookup:
         named = find_named_unit(unit, rtol=lookup_rtol)

sasdata/temp_hdf5_reader.py

@@ -9,7 +9,7 @@
 from sasdata.data import SasData
 from sasdata.data_backing import Dataset as SASDataDataset
 from sasdata.data_backing import Group as SASDataGroup
-from sasdata.dataset_types import one_dim
+from sasdata.dataset_types import one_dim, two_dim
 from sasdata.metadata import (
     Aperture,
     BeamSize,
@@ -86,7 +86,7 @@ def connected_data(node: SASDataGroup, name_prefix="") -> dict[str, Quantity]:
     for name in node.children:
         child = node.children[name]
 
-        if "units" in child.attributes:
+        if "units" in child.attributes and child.attributes["units"]:
             units = parse(child.attributes["units"])
         else:
             units = GET_UNITS_FROM_ELSEWHERE
@@ -314,24 +314,28 @@ def load_data(filename: str) -> dict[str, SasData]:
 
             data_contents : dict[str, Quantity] = {}
 
-            entry_keys = [key for key in entry if "entry" in key]
+            entry_keys = entry
 
-            if "sasdata" not in entry_keys and "data" not in entry_keys:
+            if not [k for k in entry if k.startswith("sasdata") or k.startswith("data")]:
                 logger.warning("No sasdata or data key")
+                logger.warning(f"Known keys: {[k for k in entry_keys]}")
 
             for key in entry_keys:
                 component = entry[key]
                 lower_key = key.lower()
-                if lower_key == "sasdata" or lower_key == "data":
+                if lower_key.startswith("sasdata") or lower_key.startswith("data"):
                     datum = recurse_hdf5(component)
-                    # TODO: Use named identifier
-                    data_contents = connected_data(datum, "FILE_ID_HERE")
+                    data_contents = connected_data(datum, str(filename))
 
             metadata = parse_metadata(f[root_key])
 
-            loaded_data[root_key] = SasData(
+            dataset_type = two_dim if "Qy" in data_contents else one_dim
+
+            entry_key = entry.attrs["sasview_key"] if "sasview_key" in entry.attrs else root_key
+
+            loaded_data[entry_key] = SasData(
                     name=root_key,
-                    dataset_type=one_dim,
+                    dataset_type=dataset_type,
                     data_contents=data_contents,
                     metadata=metadata,
                     verbose=False,