From d9a99c64ed0efd335efef0588fd979d3b28f5f25 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Tue, 26 May 2026 16:53:10 +0200 Subject: [PATCH 1/5] cleanup output and increase skip variables --- engine/check.py | 11 +++--- tests/util/test_log_handler.py | 61 ++++++++++++++++++++++++++++++++++ util/fof_utils.py | 3 +- util/log_handler.py | 18 ++++++++++ 4 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 tests/util/test_log_handler.py diff --git a/engine/check.py b/engine/check.py index 92c8b4e3..03b8326e 100644 --- a/engine/check.py +++ b/engine/check.py @@ -13,7 +13,7 @@ from util.click_util import CommaSeparatedStrings, cli_help from util.dataframe_ops import check_file_with_tolerances, compute_division -from util.log_handler import logger +from util.log_handler import logger, log_dataframe from util.utils import FileInfo, expand_fof @@ -68,12 +68,9 @@ def check( logger.info("RESULT: check PASSED for %s", current_file) else: logger.info("RESULT: check FAILED for %s", current_file) - logger.info("Differences") - logger.info(err) - logger.info("\nTolerance") - logger.info(tol) - logger.info("\nError relative to tolerance") - logger.info(compute_division(err, tol)) + log_dataframe(logger, "Differences", err) + log_dataframe(logger, "\nTolerances", tol) + log_dataframe(logger, "\nError relative to tolerance", compute_division(err, tol)) all_out = False sys.exit(0 if all_out else 1) diff --git a/tests/util/test_log_handler.py b/tests/util/test_log_handler.py new file mode 100644 index 00000000..a7cbe5b4 --- /dev/null +++ b/tests/util/test_log_handler.py @@ -0,0 +1,61 @@ +import logging +import pandas as pd +import pytest + +from util.log_handler import log_dataframe + + +class DummyLogger: + def __init__(self): + self.messages = [] + + def info(self, msg, *args): + # mimic logging formatting behavior + if args: + msg = msg % args + self.messages.append(msg) + + +def test_log_dataframe_none(): + logger = DummyLogger() + + log_dataframe(logger, "TITLE", None) + + assert logger.messages == [] + + +def test_log_dataframe_empty(): + logger = DummyLogger() + df = pd.DataFrame() + + log_dataframe(logger, "TITLE", df) + + assert logger.messages == [] + + +def test_log_dataframe_non_empty(): + logger = DummyLogger() + df = pd.DataFrame({"a": [1, 2], "b": [3, 4]}) + + log_dataframe(logger, "MY TITLE", df) + + assert logger.messages[0] == "MY TITLE" + + # second message is the formatted dataframe + assert "a" in logger.messages[1] + assert "1" in logger.messages[1] + assert "2" in logger.messages[1] + + +def test_log_dataframe_calls_to_string_format(): + logger = DummyLogger() + + class SpyDF(pd.DataFrame): + def to_string(self, *args, **kwargs): + return "SPY_OUTPUT" + + df = SpyDF({"a": [1]}) + + log_dataframe(logger, "TITLE", df) + + assert logger.messages[1] == "SPY_OUTPUT" \ No newline at end of file diff --git a/util/fof_utils.py b/util/fof_utils.py index de7eae9c..dc00e81a 100644 --- a/util/fof_utils.py +++ b/util/fof_utils.py @@ -181,7 +181,8 @@ def compare_var_and_attr_ds(ds1, ds2, detailed_logger): """ total_all, equal_all = 0, 0 - list_to_skip = ["source", "i_body", "l_body", "veri_data"] + list_to_skip = ["source", "i_body", "l_body", "veri_data", "record"] + list_to_skip += ["r_state", "r_check", "r_flags", "state", "check", "flags"] # temporarily skipping until tested fortran code is reproducible for var in set(ds1.data_vars).union(ds2.data_vars): if var in ds1.data_vars and var in ds2.data_vars and var not in list_to_skip: diff --git a/util/log_handler.py b/util/log_handler.py index f66bb241..e91a2595 100644 --- a/util/log_handler.py +++ b/util/log_handler.py @@ -8,6 +8,8 @@ import logging import sys +from typing import Optional +import pandas as pd mpl_logger = logging.getLogger("matplotlib") mpl_logger.setLevel(logging.WARNING) @@ -57,3 +59,19 @@ def initialize_detailed_logger( detailed_logger.info("initialized named logger '%s'", name) return detailed_logger + +def log_dataframe( + logger: logging.Logger, + title: str, + df: Optional[pd.DataFrame], +) -> None: + """ + Log a DataFrame only if it contains data. + + Avoids printing empty pandas DataFrames in CI/log outputs. + """ + if df is None or df.empty: + return + + logger.info(title) + logger.info("%s", df.to_string(index=False)) \ No newline at end of file From 984837124ca80df237bbcad3e5d6c6dc930265d1 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Tue, 26 May 2026 17:27:58 +0200 Subject: [PATCH 2/5] fix pytest --- engine/check.py | 6 ++++-- tests/util/test_fof_utils.py | 4 +--- tests/util/test_log_handler.py | 20 +++++++++++++------- util/fof_utils.py | 9 ++++++++- util/log_handler.py | 10 +++++----- 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/engine/check.py b/engine/check.py index 03b8326e..c75ad5be 100644 --- a/engine/check.py +++ b/engine/check.py @@ -13,7 +13,7 @@ from util.click_util import CommaSeparatedStrings, cli_help from util.dataframe_ops import check_file_with_tolerances, compute_division -from util.log_handler import logger, log_dataframe +from util.log_handler import log_dataframe, logger from util.utils import FileInfo, expand_fof @@ -70,7 +70,9 @@ def check( logger.info("RESULT: check FAILED for %s", current_file) log_dataframe(logger, "Differences", err) log_dataframe(logger, "\nTolerances", tol) - log_dataframe(logger, "\nError relative to tolerance", compute_division(err, tol)) + log_dataframe( + logger, "\nError relative to tolerance", compute_division(err, tol) + ) all_out = False sys.exit(0 if all_out else 1) diff --git a/tests/util/test_fof_utils.py b/tests/util/test_fof_utils.py index 77948691..2f42cda3 100644 --- a/tests/util/test_fof_utils.py +++ b/tests/util/test_fof_utils.py @@ -249,10 +249,8 @@ def test_compare_var_and_attr_ds(ds1, ds2): ) total1, equal1 = compare_var_and_attr_ds(ds1, ds2, detailed_logger) - total2, equal2 = compare_var_and_attr_ds(ds1, ds2, detailed_logger) - assert (total1, equal1) == (103, 102) - assert (total2, equal2) == (103, 102) + assert (total1, equal1) == (85, 84) @pytest.fixture(name="ds3") diff --git a/tests/util/test_log_handler.py b/tests/util/test_log_handler.py index a7cbe5b4..2f075f0c 100644 --- a/tests/util/test_log_handler.py +++ b/tests/util/test_log_handler.py @@ -1,11 +1,15 @@ -import logging +""" +This module contains unit tests for the `util/log_handler.py` module. +""" + import pandas as pd -import pytest from util.log_handler import log_dataframe -class DummyLogger: +class DummyLogger: # pylint: disable=too-few-public-methods + """Simple logger that stores messages in memory for testing.""" + def __init__(self): self.messages = [] @@ -21,7 +25,7 @@ def test_log_dataframe_none(): log_dataframe(logger, "TITLE", None) - assert logger.messages == [] + assert not logger.messages def test_log_dataframe_empty(): @@ -30,7 +34,7 @@ def test_log_dataframe_empty(): log_dataframe(logger, "TITLE", df) - assert logger.messages == [] + assert not logger.messages def test_log_dataframe_non_empty(): @@ -51,11 +55,13 @@ def test_log_dataframe_calls_to_string_format(): logger = DummyLogger() class SpyDF(pd.DataFrame): - def to_string(self, *args, **kwargs): + """DataFrame subclass that overrides to_string for testing.""" + + def to_string(self, *_args, **_kwargs): return "SPY_OUTPUT" df = SpyDF({"a": [1]}) log_dataframe(logger, "TITLE", df) - assert logger.messages[1] == "SPY_OUTPUT" \ No newline at end of file + assert logger.messages[1] == "SPY_OUTPUT" diff --git a/util/fof_utils.py b/util/fof_utils.py index dc00e81a..33134026 100644 --- a/util/fof_utils.py +++ b/util/fof_utils.py @@ -182,7 +182,14 @@ def compare_var_and_attr_ds(ds1, ds2, detailed_logger): total_all, equal_all = 0, 0 list_to_skip = ["source", "i_body", "l_body", "veri_data", "record"] - list_to_skip += ["r_state", "r_check", "r_flags", "state", "check", "flags"] # temporarily skipping until tested fortran code is reproducible + list_to_skip += [ + "r_state", + "r_check", + "r_flags", + "state", + "check", + "flags", + ] # temporarily skipping until tested fortran code is reproducible for var in set(ds1.data_vars).union(ds2.data_vars): if var in ds1.data_vars and var in ds2.data_vars and var not in list_to_skip: diff --git a/util/log_handler.py b/util/log_handler.py index e91a2595..3b401db0 100644 --- a/util/log_handler.py +++ b/util/log_handler.py @@ -9,6 +9,7 @@ import logging import sys from typing import Optional + import pandas as pd mpl_logger = logging.getLogger("matplotlib") @@ -60,18 +61,17 @@ def initialize_detailed_logger( detailed_logger.info("initialized named logger '%s'", name) return detailed_logger + def log_dataframe( - logger: logging.Logger, + log: logging.Logger, title: str, df: Optional[pd.DataFrame], ) -> None: """ Log a DataFrame only if it contains data. - - Avoids printing empty pandas DataFrames in CI/log outputs. """ if df is None or df.empty: return - logger.info(title) - logger.info("%s", df.to_string(index=False)) \ No newline at end of file + log.info(title) + log.info("%s", df.to_string(index=False)) From 2a9093931d4f9d908a1a197c495cdc01dfbc45b2 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Tue, 26 May 2026 17:41:42 +0200 Subject: [PATCH 3/5] fix test --- tests/engine/test_fof_compare.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/engine/test_fof_compare.py b/tests/engine/test_fof_compare.py index e51fc25d..0781dd82 100644 --- a/tests/engine/test_fof_compare.py +++ b/tests/engine/test_fof_compare.py @@ -21,6 +21,7 @@ def fixture_fof_datasets(fof_datasets_base, tmp_dir): ds1, ds2, _, _ = fof_datasets_base ds3 = ds2.copy(deep=True) ds3["flags"] = (("d_body",), ds3["flags"].values * 1.55) + ds3["e_o"] = (("d_body",), ds3["e_o"].values * 1.55) ds1_file = os.path.join(tmp_dir, "fof1_SYNOP.nc") ds2_file = os.path.join(tmp_dir, "fof2_SYNOP.nc") From b1b844f44617b7daa136cacd59b1940b8bc7ea69 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Tue, 26 May 2026 17:53:19 +0200 Subject: [PATCH 4/5] revert unclean fix --- tests/util/test_fof_utils.py | 2 +- util/fof_utils.py | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/util/test_fof_utils.py b/tests/util/test_fof_utils.py index 2f42cda3..57e60d08 100644 --- a/tests/util/test_fof_utils.py +++ b/tests/util/test_fof_utils.py @@ -250,7 +250,7 @@ def test_compare_var_and_attr_ds(ds1, ds2): total1, equal1 = compare_var_and_attr_ds(ds1, ds2, detailed_logger) - assert (total1, equal1) == (85, 84) + assert (total1, equal1) == (103, 102) @pytest.fixture(name="ds3") diff --git a/util/fof_utils.py b/util/fof_utils.py index 33134026..05361d75 100644 --- a/util/fof_utils.py +++ b/util/fof_utils.py @@ -182,14 +182,6 @@ def compare_var_and_attr_ds(ds1, ds2, detailed_logger): total_all, equal_all = 0, 0 list_to_skip = ["source", "i_body", "l_body", "veri_data", "record"] - list_to_skip += [ - "r_state", - "r_check", - "r_flags", - "state", - "check", - "flags", - ] # temporarily skipping until tested fortran code is reproducible for var in set(ds1.data_vars).union(ds2.data_vars): if var in ds1.data_vars and var in ds2.data_vars and var not in list_to_skip: From 3fcef5e066109b1fdc92cbaed24c4357d04a68d2 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Wed, 27 May 2026 08:55:48 +0200 Subject: [PATCH 5/5] revert other changes --- engine/check.py | 13 ++++--- tests/engine/test_fof_compare.py | 1 - tests/util/test_log_handler.py | 67 -------------------------------- util/log_handler.py | 18 --------- 4 files changed, 7 insertions(+), 92 deletions(-) delete mode 100644 tests/util/test_log_handler.py diff --git a/engine/check.py b/engine/check.py index c75ad5be..92c8b4e3 100644 --- a/engine/check.py +++ b/engine/check.py @@ -13,7 +13,7 @@ from util.click_util import CommaSeparatedStrings, cli_help from util.dataframe_ops import check_file_with_tolerances, compute_division -from util.log_handler import log_dataframe, logger +from util.log_handler import logger from util.utils import FileInfo, expand_fof @@ -68,11 +68,12 @@ def check( logger.info("RESULT: check PASSED for %s", current_file) else: logger.info("RESULT: check FAILED for %s", current_file) - log_dataframe(logger, "Differences", err) - log_dataframe(logger, "\nTolerances", tol) - log_dataframe( - logger, "\nError relative to tolerance", compute_division(err, tol) - ) + logger.info("Differences") + logger.info(err) + logger.info("\nTolerance") + logger.info(tol) + logger.info("\nError relative to tolerance") + logger.info(compute_division(err, tol)) all_out = False sys.exit(0 if all_out else 1) diff --git a/tests/engine/test_fof_compare.py b/tests/engine/test_fof_compare.py index 0781dd82..e51fc25d 100644 --- a/tests/engine/test_fof_compare.py +++ b/tests/engine/test_fof_compare.py @@ -21,7 +21,6 @@ def fixture_fof_datasets(fof_datasets_base, tmp_dir): ds1, ds2, _, _ = fof_datasets_base ds3 = ds2.copy(deep=True) ds3["flags"] = (("d_body",), ds3["flags"].values * 1.55) - ds3["e_o"] = (("d_body",), ds3["e_o"].values * 1.55) ds1_file = os.path.join(tmp_dir, "fof1_SYNOP.nc") ds2_file = os.path.join(tmp_dir, "fof2_SYNOP.nc") diff --git a/tests/util/test_log_handler.py b/tests/util/test_log_handler.py deleted file mode 100644 index 2f075f0c..00000000 --- a/tests/util/test_log_handler.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -This module contains unit tests for the `util/log_handler.py` module. -""" - -import pandas as pd - -from util.log_handler import log_dataframe - - -class DummyLogger: # pylint: disable=too-few-public-methods - """Simple logger that stores messages in memory for testing.""" - - def __init__(self): - self.messages = [] - - def info(self, msg, *args): - # mimic logging formatting behavior - if args: - msg = msg % args - self.messages.append(msg) - - -def test_log_dataframe_none(): - logger = DummyLogger() - - log_dataframe(logger, "TITLE", None) - - assert not logger.messages - - -def test_log_dataframe_empty(): - logger = DummyLogger() - df = pd.DataFrame() - - log_dataframe(logger, "TITLE", df) - - assert not logger.messages - - -def test_log_dataframe_non_empty(): - logger = DummyLogger() - df = pd.DataFrame({"a": [1, 2], "b": [3, 4]}) - - log_dataframe(logger, "MY TITLE", df) - - assert logger.messages[0] == "MY TITLE" - - # second message is the formatted dataframe - assert "a" in logger.messages[1] - assert "1" in logger.messages[1] - assert "2" in logger.messages[1] - - -def test_log_dataframe_calls_to_string_format(): - logger = DummyLogger() - - class SpyDF(pd.DataFrame): - """DataFrame subclass that overrides to_string for testing.""" - - def to_string(self, *_args, **_kwargs): - return "SPY_OUTPUT" - - df = SpyDF({"a": [1]}) - - log_dataframe(logger, "TITLE", df) - - assert logger.messages[1] == "SPY_OUTPUT" diff --git a/util/log_handler.py b/util/log_handler.py index 3b401db0..f66bb241 100644 --- a/util/log_handler.py +++ b/util/log_handler.py @@ -8,9 +8,6 @@ import logging import sys -from typing import Optional - -import pandas as pd mpl_logger = logging.getLogger("matplotlib") mpl_logger.setLevel(logging.WARNING) @@ -60,18 +57,3 @@ def initialize_detailed_logger( detailed_logger.info("initialized named logger '%s'", name) return detailed_logger - - -def log_dataframe( - log: logging.Logger, - title: str, - df: Optional[pd.DataFrame], -) -> None: - """ - Log a DataFrame only if it contains data. - """ - if df is None or df.empty: - return - - log.info(title) - log.info("%s", df.to_string(index=False))