From 536b80fee040c22a3789c149f68d6bfc2178e534 Mon Sep 17 00:00:00 2001 From: NITISH-R-G <225521762+NITISH-R-G@users.noreply.github.com> Date: Fri, 19 Jun 2026 08:35:57 +0000 Subject: [PATCH 1/2] Add tests for evaluation/main.py Created code/tests/test_evaluation_main.py covering normalization, accuracy calculation, and strategy selection. Also implemented the corresponding core logic in code/evaluation/main.py. Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com> --- code/evaluation/main.py | 43 +++++++++++++++++++++++++++++ code/tests/test_evaluation_main.py | 44 ++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 code/tests/test_evaluation_main.py diff --git a/code/evaluation/main.py b/code/evaluation/main.py index e69de29b..02bdb8a9 100644 --- a/code/evaluation/main.py +++ b/code/evaluation/main.py @@ -0,0 +1,43 @@ +from typing import List, Dict, Any + +def normalize_value(value: Any) -> str: + """ + Normalizes a value for comparison. + Converts to string, lowercases, and strips whitespace. + Booleans are converted to their lowercase string representation. + None is converted to 'none'. + """ + if value is None: + return "none" + if isinstance(value, bool): + return str(value).lower() + return str(value).strip().lower() + +def calculate_accuracy(expected: List[str], predicted: List[str]) -> float: + """ + Calculates exact-match accuracy between two lists of equal length. + Returns 0.0 if the lists are empty. + Raises ValueError if lists are of different lengths. + """ + if len(expected) != len(predicted): + raise ValueError("Expected and predicted lists must be of the same length.") + + if not expected: + return 0.0 + + matches = sum(1 for e, p in zip(expected, predicted) if normalize_value(e) == normalize_value(p)) + return matches / len(expected) + +def select_winning_strategy(metrics: Dict[str, Dict[str, float]]) -> str: + """ + Selects the winning strategy based on the mean accuracy. + In case of a tie, prefers Strategy B (two-pass) for determinism. + metrics should be like: {"strategy_a": {"mean": 0.8}, "strategy_b": {"mean": 0.9}} + """ + mean_a = metrics.get("strategy_a", {}).get("mean", 0.0) + mean_b = metrics.get("strategy_b", {}).get("mean", 0.0) + + if mean_a > mean_b: + return "strategy_a" + else: + return "strategy_b" diff --git a/code/tests/test_evaluation_main.py b/code/tests/test_evaluation_main.py new file mode 100644 index 00000000..94d13b32 --- /dev/null +++ b/code/tests/test_evaluation_main.py @@ -0,0 +1,44 @@ +import pytest +from evaluation.main import normalize_value, calculate_accuracy, select_winning_strategy + +def test_normalize_value(): + assert normalize_value(" Hello ") == "hello" + assert normalize_value("HELLO") == "hello" + assert normalize_value("hello") == "hello" + assert normalize_value(True) == "true" + assert normalize_value(False) == "false" + assert normalize_value(None) == "none" + +def test_calculate_accuracy(): + # Exactly matching lists + expected = ["supported", "dent", "door", "true", "none"] + predicted = ["supported", "dent", "door", "true", "none"] + assert calculate_accuracy(expected, predicted) == 1.0 + + # Partial match + predicted_partial = ["supported", "scratch", "door", "false", "none"] + assert calculate_accuracy(expected, predicted_partial) == 3 / 5 + + # Completely wrong + predicted_wrong = ["contradicted", "scratch", "hood", "false", "unknown"] + assert calculate_accuracy(expected, predicted_wrong) == 0.0 + + # Empty lists + assert calculate_accuracy([], []) == 0.0 + + # Different lengths should raise ValueError + with pytest.raises(ValueError): + calculate_accuracy(["a", "b"], ["a"]) + +def test_select_winning_strategy(): + # Strategy A wins + metrics_a_wins = {"strategy_a": {"mean": 0.9}, "strategy_b": {"mean": 0.8}} + assert select_winning_strategy(metrics_a_wins) == "strategy_a" + + # Strategy B wins + metrics_b_wins = {"strategy_a": {"mean": 0.7}, "strategy_b": {"mean": 0.8}} + assert select_winning_strategy(metrics_b_wins) == "strategy_b" + + # Tie - B should win + metrics_tie = {"strategy_a": {"mean": 0.85}, "strategy_b": {"mean": 0.85}} + assert select_winning_strategy(metrics_tie) == "strategy_b" From d08d830ec91a3dd91d3361743913bbb82610a5b8 Mon Sep 17 00:00:00 2001 From: NITISH-R-G <225521762+NITISH-R-G@users.noreply.github.com> Date: Fri, 19 Jun 2026 08:38:04 +0000 Subject: [PATCH 2/2] Use pytest.approx for floating point comparisons in test_evaluation_main.py Replaced exact equality checks with pytest.approx for floating point calculations in accuracy tests to avoid precision errors and fix SonarCloud CI warnings. Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com> --- code/tests/test_evaluation_main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/code/tests/test_evaluation_main.py b/code/tests/test_evaluation_main.py index 94d13b32..354e5f08 100644 --- a/code/tests/test_evaluation_main.py +++ b/code/tests/test_evaluation_main.py @@ -13,18 +13,18 @@ def test_calculate_accuracy(): # Exactly matching lists expected = ["supported", "dent", "door", "true", "none"] predicted = ["supported", "dent", "door", "true", "none"] - assert calculate_accuracy(expected, predicted) == 1.0 + assert calculate_accuracy(expected, predicted) == pytest.approx(1.0) # Partial match predicted_partial = ["supported", "scratch", "door", "false", "none"] - assert calculate_accuracy(expected, predicted_partial) == 3 / 5 + assert calculate_accuracy(expected, predicted_partial) == pytest.approx(0.6) # Completely wrong predicted_wrong = ["contradicted", "scratch", "hood", "false", "unknown"] - assert calculate_accuracy(expected, predicted_wrong) == 0.0 + assert calculate_accuracy(expected, predicted_wrong) == pytest.approx(0.0) # Empty lists - assert calculate_accuracy([], []) == 0.0 + assert calculate_accuracy([], []) == pytest.approx(0.0) # Different lengths should raise ValueError with pytest.raises(ValueError):