🧪 Add tests and core logic for evaluation/main.py

code/evaluation/main.py

@@ -0,0 +1,43 @@
+from typing import List, Dict, Any
+
+def normalize_value(value: Any) -> str:
+    """
+    Normalizes a value for comparison.
+    Converts to string, lowercases, and strips whitespace.
+    Booleans are converted to their lowercase string representation.
+    None is converted to 'none'.
+    """
+    if value is None:
+        return "none"
+    if isinstance(value, bool):
+        return str(value).lower()
+    return str(value).strip().lower()
+
+def calculate_accuracy(expected: List[str], predicted: List[str]) -> float:
+    """
+    Calculates exact-match accuracy between two lists of equal length.
+    Returns 0.0 if the lists are empty.
+    Raises ValueError if lists are of different lengths.
+    """
+    if len(expected) != len(predicted):
+        raise ValueError("Expected and predicted lists must be of the same length.")
+
+    if not expected:
+        return 0.0
+
+    matches = sum(1 for e, p in zip(expected, predicted) if normalize_value(e) == normalize_value(p))
+    return matches / len(expected)
+
+def select_winning_strategy(metrics: Dict[str, Dict[str, float]]) -> str:
+    """
+    Selects the winning strategy based on the mean accuracy.
+    In case of a tie, prefers Strategy B (two-pass) for determinism.
+    metrics should be like: {"strategy_a": {"mean": 0.8}, "strategy_b": {"mean": 0.9}}
+    """
+    mean_a = metrics.get("strategy_a", {}).get("mean", 0.0)
+    mean_b = metrics.get("strategy_b", {}).get("mean", 0.0)
+
+    if mean_a > mean_b:
+        return "strategy_a"
+    else:
+        return "strategy_b"

code/tests/test_evaluation_main.py

@@ -0,0 +1,44 @@
+import pytest
+from evaluation.main import normalize_value, calculate_accuracy, select_winning_strategy
+
+def test_normalize_value():
+    assert normalize_value("  Hello  ") == "hello"
+    assert normalize_value("HELLO") == "hello"
+    assert normalize_value("hello") == "hello"
+    assert normalize_value(True) == "true"
+    assert normalize_value(False) == "false"
+    assert normalize_value(None) == "none"
+
+def test_calculate_accuracy():
+    # Exactly matching lists
+    expected = ["supported", "dent", "door", "true", "none"]
+    predicted = ["supported", "dent", "door", "true", "none"]
+    assert calculate_accuracy(expected, predicted) == pytest.approx(1.0)
+
+    # Partial match
+    predicted_partial = ["supported", "scratch", "door", "false", "none"]
+    assert calculate_accuracy(expected, predicted_partial) == pytest.approx(0.6)
+
+    # Completely wrong
+    predicted_wrong = ["contradicted", "scratch", "hood", "false", "unknown"]
+    assert calculate_accuracy(expected, predicted_wrong) == pytest.approx(0.0)
+
+    # Empty lists
+    assert calculate_accuracy([], []) == pytest.approx(0.0)
+
+    # Different lengths should raise ValueError
+    with pytest.raises(ValueError):
+        calculate_accuracy(["a", "b"], ["a"])
+
+def test_select_winning_strategy():
+    # Strategy A wins
+    metrics_a_wins = {"strategy_a": {"mean": 0.9}, "strategy_b": {"mean": 0.8}}
+    assert select_winning_strategy(metrics_a_wins) == "strategy_a"
+
+    # Strategy B wins
+    metrics_b_wins = {"strategy_a": {"mean": 0.7}, "strategy_b": {"mean": 0.8}}
+    assert select_winning_strategy(metrics_b_wins) == "strategy_b"
+
+    # Tie - B should win
+    metrics_tie = {"strategy_a": {"mean": 0.85}, "strategy_b": {"mean": 0.85}}
+    assert select_winning_strategy(metrics_tie) == "strategy_b"