Merge branch 'main' of github.com:comet-toolkit/obsarray into main

Author: pdevis

obsarray/templater/dataset_util.py

@@ -153,7 +153,7 @@ def create_unc_variable(
         # set undefined dims as random
         defined_err_corr_dims = []
         for erd in err_corr:
-            if isinstance(erd["dim"],str):
+            if isinstance(erd["dim"], str):
                 defined_err_corr_dims.append(erd["dim"])
             else:
                 defined_err_corr_dims.extend(erd["dim"])
@@ -173,7 +173,10 @@ def create_unc_variable(
             units_str = DatasetUtil.return_err_corr_units_str(idx)
 
             form = ecdef["form"]
-            attributes[dim_str] = ecdef["dim"]
+            if isinstance(ecdef["dim"], list) and len(ecdef["dim"]) == 1:
+                attributes[dim_str] = ecdef["dim"]
+            else:
+                attributes[dim_str] = ecdef["dim"]
             attributes[form_str] = ecdef["form"]
             attributes[units_str] = ecdef["units"] if "units" in ecdef else []
 
@@ -383,7 +386,9 @@ def unpack_flags(da):
 
         ds = xarray.Dataset()
         for flag_meaning, flag_mask in zip(flag_meanings, flag_masks):
-            ds[flag_meaning] = DatasetUtil.create_variable(list(da.shape), bool, dim_names=list(da.dims))
+            ds[flag_meaning] = DatasetUtil.create_variable(
+                list(da.shape), bool, dim_names=list(da.dims)
+            )
             ds[flag_meaning] = (da & flag_mask).astype(bool)
 
         return ds
@@ -447,7 +452,9 @@ def set_flag(da, flag_name, error_if_set=False):
         set_flags = DatasetUtil.unpack_flags(da)[flag_name]
 
         if numpy.any(set_flags == True) and error_if_set:
-            raise ValueError("Flag " + flag_name + " already set for variable " + da.name)
+            raise ValueError(
+                "Flag " + flag_name + " already set for variable " + da.name
+            )
 
         # Find flag mask
         flag_meanings, flag_masks = DatasetUtil._get_flag_encoding(da)
@@ -473,7 +480,9 @@ def unset_flag(da, flag_name, error_if_unset=False):
         set_flags = DatasetUtil.unpack_flags(da)[flag_name]
 
         if numpy.any(set_flags == False) and error_if_unset:
-            raise ValueError("Flag " + flag_name + " already set for variable " + da.name)
+            raise ValueError(
+                "Flag " + flag_name + " already set for variable " + da.name
+            )
 
         # Find flag mask
         flag_meanings, flag_masks = DatasetUtil._get_flag_encoding(da)
@@ -501,7 +510,7 @@ def get_set_flags(da):
 
         set_flags = []
         for flag_meaning, flag_mask in zip(flag_meanings, flag_masks):
-            if (da & flag_mask):
+            if da & flag_mask:
                 set_flags.append(flag_meaning)
 
         return set_flags

obsarray/templater/template_util.py

@@ -11,15 +11,19 @@
 
 
 def create_ds(
-    template: Dict[str, Dict], size: Dict[str, int], metadata: Optional[Dict] = None,
-        propagate_ds: Optional[xarray.Dataset] = None,
+    template: Dict[str, Dict],
+    size: Dict[str, int],
+    metadata: Optional[Dict] = None,
+    append_ds: Optional[xarray.Dataset] = None,
+    propagate_ds: Optional[xarray.Dataset] = None,
 ) -> xarray.Dataset:
     """
     Returns template dataset
 
     :param template: dictionary defining ds variable structure, as defined below.
     :param size: dictionary of dataset dimensions, entry per dataset dimension with value of size as int
     :param metadata: dictionary of dataset metadata
+    :param append_ds: base dataset to append with template variables
     :param propagate_ds: template dataset is populated with data from propagate_ds for their variables with
     common names and dimensions. Useful for transferring common data between datasets at different processing levels
     (e.g. times, etc.).
@@ -38,7 +42,7 @@ def create_ds(
     """
 
     # Create dataset
-    ds = xarray.Dataset()
+    ds = append_ds if append_ds is not None else xarray.Dataset()
 
     # Add variables
     ds = TemplateUtil.add_variables(ds, template, size)
@@ -85,7 +89,9 @@ def add_variables(
 
         :returns: dataset with defined variables
         """
+
         for var_name in template.keys():
+
             var = TemplateUtil._create_var(var_name, template[var_name], size)
 
             ds[var_name] = var
@@ -123,7 +129,7 @@ def _create_var(
             )
 
         # Create variable and add to dataset
-        if isinstance(dtype,str):
+        if dtype == str:
             if dtype == "flag":
                 flag_meanings = attributes.pop("flag_meanings")
                 variable = du.create_flags_variable(
@@ -218,21 +224,32 @@ def propagate_values(target_ds, source_ds, exclude=None):
 
         # Find variable names common to target_ds and source_ds, excluding specified exclude variables
         common_variable_names = list(set(target_ds).intersection(source_ds))
-        #common_variable_names = list(set(target_ds.variables).intersection(source_ds.variables))
-        #print(common_variable_names)
+        # common_variable_names = list(set(target_ds.variables).intersection(source_ds.variables))
+        # print(common_variable_names)
 
         if exclude is not None:
-            common_variable_names = [name for name in common_variable_names if name not in exclude]
+            common_variable_names = [
+                name for name in common_variable_names if name not in exclude
+            ]
 
         # Remove any common variables that have different dimensions in target_ds and source_ds
-        common_variable_names = [name for name in common_variable_names if target_ds[name].dims == source_ds[name].dims]
+        common_variable_names = [
+            name
+            for name in common_variable_names
+            if target_ds[name].dims == source_ds[name].dims
+        ]
 
         # Propagate data
         for common_variable_name in common_variable_names:
-            if target_ds[common_variable_name].shape == source_ds[common_variable_name].shape:
-                target_ds[common_variable_name].values = source_ds[common_variable_name].values
-
-        #to do - add method to propagate common unpopulated metadata
+            if (
+                target_ds[common_variable_name].shape
+                == source_ds[common_variable_name].shape
+            ):
+                target_ds[common_variable_name].values = source_ds[
+                    common_variable_name
+                ].values
+
+        # to do - add method to propagate common unpopulated metadata
 
 
 if __name__ == "__main__":

obsarray/templater/tests/test_dataset_util.py

@@ -173,7 +173,36 @@ def test_create_variable_3D_int_attributes(self):
         self.assertEqual(-127, array_variable[2, 4, 2])
         self.assertEqual("std", array_variable.attrs["standard_name"])
 
-    def test_create_unc_variable(self):
+    def test_create_unc_variable_1dimundef(self):
+        err_corr = [
+            {
+                "dim": ["x"],
+                "form": "rectangle_absolute",
+                "params": [1, 2],
+                "units": ["m", "m"],
+            },
+        ]
+
+        unc_variable = DatasetUtil.create_unc_variable(
+            [7],
+            np.int8,
+            ["x"],
+            pdf_shape="gaussian",
+            err_corr=err_corr,
+        )
+
+        expected_attrs = {
+            "err_corr_1_dim": ["x"],
+            "err_corr_1_form": "rectangle_absolute",
+            "err_corr_1_units": ["m", "m"],
+            "err_corr_1_params": [1, 2],
+        }
+
+        actual_attrs = unc_variable.attrs
+
+        self.assertTrue(expected_attrs.items() <= actual_attrs.items())
+
+    def test_create_unc_variable_1undef(self):
         err_corr = [
             {
                 "dim": "x",

obsarray/templater/tests/test_template_util.py

@@ -285,6 +285,50 @@ def test_create_template_dataset(self):
         self.assertEqual(type(ds["array_variable"]), xarray.DataArray)
         self.assertEqual("value", ds.attrs["metadata1"])
 
+    def test_create_template_dataset_withunc(self):
+        dim_sizes = {"dim1": 25}
+
+        test_variables = {
+            "array_variable": {
+                "dim": ["dim1"],
+                "dtype": np.float32,
+                "attributes": {
+                    "standard_name": "array_variable_std_name",
+                    "long_name": "array_variable_long_name",
+                    "units": "units",
+                    "preferred_symbol": "av",
+                    "unc_comps": ["u_array_variable"],
+                },
+                "encoding": {"dtype": np.uint16, "scale_factor": 1.0, "offset": 0.0},
+            },
+            "u_array_variable": {
+                "dim": ["dim1"],
+                "dtype": np.float32,
+                "attributes": {
+                    "err_corr": [
+                        {
+                            "dim": ["dim1"],
+                            "form": "rectangle_absolute",
+                            "params": [1, 2],
+                            "units": ["m", "m"],
+                        },
+                    ],
+                    "standard_name": "array_variable_std_name",
+                    "long_name": "array_variable_long_name",
+                    "units": "units",
+                    "preferred_symbol": "av",
+                },
+            },
+        }
+
+        test_metadata = {"metadata1": "value"}
+
+        ds = create_ds(test_variables, dim_sizes, test_metadata)
+
+        self.assertEqual(type(ds), xarray.Dataset)
+        self.assertEqual(type(ds["array_variable"]), xarray.DataArray)
+        self.assertEqual("value", ds.attrs["metadata1"])
+
 
 if __name__ == "__main__":
     unittest.main()

obsarray/test/test_unc_accessor.py

@@ -648,6 +648,15 @@ def unitcall(ds):
 
         self.assertRaises(ValueError, unitcall, self.ds)
 
+    def test_abs_value_uncnounits(self):
+
+        del self.ds["u_str_temperature"].attrs["units"]
+
+        xr.testing.assert_equal(
+            self.ds.unc["temperature"]["u_str_temperature"].value,
+            self.ds.unc["temperature"]["u_str_temperature"].abs_value,
+        )
+
     def test_pdf_shape(self):
         self.assertEqual(
             self.ds.unc["temperature"]["u_ran_temperature"].pdf_shape, "gaussian"

obsarray/unc_accessor.py

@@ -219,7 +219,7 @@ def abs_value(self) -> xr.DataArray:
             if self.units == "%":
                 return self.value / 100 * self.var_value
 
-            elif self.units != self.var_units:
+            elif (self.units != self.var_units) and (self.units is not None):
                 raise ValueError(
                     "Unit mismatch between observation variable and uncertainty variable:\n"
                     "* '{}' - '{}'\n"
@@ -246,7 +246,7 @@ def is_random(self) -> bool:
 
         :return: random uncertainty flag
         """
-        if len(self.err_corr)>0:
+        if len(self.err_corr) > 0:
             return all(e[1].is_random is True for e in self.err_corr)
         else:
             return False