diff --git a/code/evaluation/main.py b/code/evaluation/main.py index e69de29b..02bdb8a9 100644 --- a/code/evaluation/main.py +++ b/code/evaluation/main.py @@ -0,0 +1,43 @@ +from typing import List, Dict, Any + +def normalize_value(value: Any) -> str: + """ + Normalizes a value for comparison. + Converts to string, lowercases, and strips whitespace. + Booleans are converted to their lowercase string representation. + None is converted to 'none'. + """ + if value is None: + return "none" + if isinstance(value, bool): + return str(value).lower() + return str(value).strip().lower() + +def calculate_accuracy(expected: List[str], predicted: List[str]) -> float: + """ + Calculates exact-match accuracy between two lists of equal length. + Returns 0.0 if the lists are empty. + Raises ValueError if lists are of different lengths. + """ + if len(expected) != len(predicted): + raise ValueError("Expected and predicted lists must be of the same length.") + + if not expected: + return 0.0 + + matches = sum(1 for e, p in zip(expected, predicted) if normalize_value(e) == normalize_value(p)) + return matches / len(expected) + +def select_winning_strategy(metrics: Dict[str, Dict[str, float]]) -> str: + """ + Selects the winning strategy based on the mean accuracy. + In case of a tie, prefers Strategy B (two-pass) for determinism. + metrics should be like: {"strategy_a": {"mean": 0.8}, "strategy_b": {"mean": 0.9}} + """ + mean_a = metrics.get("strategy_a", {}).get("mean", 0.0) + mean_b = metrics.get("strategy_b", {}).get("mean", 0.0) + + if mean_a > mean_b: + return "strategy_a" + else: + return "strategy_b" diff --git a/code/tests/test_evaluation_main.py b/code/tests/test_evaluation_main.py new file mode 100644 index 00000000..354e5f08 --- /dev/null +++ b/code/tests/test_evaluation_main.py @@ -0,0 +1,44 @@ +import pytest +from evaluation.main import normalize_value, calculate_accuracy, select_winning_strategy + +def test_normalize_value(): + assert normalize_value(" Hello ") == "hello" + assert normalize_value("HELLO") == "hello" + assert normalize_value("hello") == "hello" + assert normalize_value(True) == "true" + assert normalize_value(False) == "false" + assert normalize_value(None) == "none" + +def test_calculate_accuracy(): + # Exactly matching lists + expected = ["supported", "dent", "door", "true", "none"] + predicted = ["supported", "dent", "door", "true", "none"] + assert calculate_accuracy(expected, predicted) == pytest.approx(1.0) + + # Partial match + predicted_partial = ["supported", "scratch", "door", "false", "none"] + assert calculate_accuracy(expected, predicted_partial) == pytest.approx(0.6) + + # Completely wrong + predicted_wrong = ["contradicted", "scratch", "hood", "false", "unknown"] + assert calculate_accuracy(expected, predicted_wrong) == pytest.approx(0.0) + + # Empty lists + assert calculate_accuracy([], []) == pytest.approx(0.0) + + # Different lengths should raise ValueError + with pytest.raises(ValueError): + calculate_accuracy(["a", "b"], ["a"]) + +def test_select_winning_strategy(): + # Strategy A wins + metrics_a_wins = {"strategy_a": {"mean": 0.9}, "strategy_b": {"mean": 0.8}} + assert select_winning_strategy(metrics_a_wins) == "strategy_a" + + # Strategy B wins + metrics_b_wins = {"strategy_a": {"mean": 0.7}, "strategy_b": {"mean": 0.8}} + assert select_winning_strategy(metrics_b_wins) == "strategy_b" + + # Tie - B should win + metrics_tie = {"strategy_a": {"mean": 0.85}, "strategy_b": {"mean": 0.85}} + assert select_winning_strategy(metrics_tie) == "strategy_b"