🧪 [Add evaluation logic and corresponding tests]

code/evaluation/main.py

@@ -0,0 +1,61 @@
+"""Metrics evaluation module."""
+import csv
+from pathlib import Path
+from typing import Dict, Union
+
+
+def evaluate_predictions(
+    predictions_path: Path, truth_path: Path
+) -> Dict[str, Union[int, float]]:
+    """Evaluate accuracy of predictions against ground truth."""
+    if not predictions_path.exists() or not truth_path.exists():
+        return {}
+
+    with open(predictions_path, 'r', encoding='utf-8') as f:
+        preds = list(csv.DictReader(f))
+    with open(truth_path, 'r', encoding='utf-8') as f:
+        truth = {row['user_id']: row for row in csv.DictReader(f)}
+
+    metrics: Dict[str, Union[int, float]] = {
+        'total': len(preds),
+        'correct_claim_status': 0,
+        'correct_issue_type': 0,
+        'correct_object_part': 0,
+        'correct_severity': 0
+    }
+
+    fields = ['claim_status', 'issue_type', 'object_part', 'severity']
+
+    for pred in preds:
+        user_id = pred.get('user_id')
+        if not user_id or user_id not in truth:
+            continue
+
+        t_row = truth[user_id]
+        for field in fields:
+            if pred.get(field) == t_row.get(field):
+                metrics[f'correct_{field}'] += 1  # type: ignore
+
+    total = int(metrics['total'])
+    for field in fields:
+        correct = int(metrics[f'correct_{field}'])
+        acc_key = f'accuracy_{field}'
+        metrics[acc_key] = correct / total if total > 0 else 0.0
+
+    return metrics
+
+
+def main() -> None:
+    """Run evaluation on sample claims."""
+    base_dir = Path(__file__).parent.parent.parent
+    preds_csv = base_dir / "output.csv"
+    truth_csv = base_dir / "dataset" / "sample_claims.csv"
+
+    if preds_csv.exists() and truth_csv.exists():
+        metrics = evaluate_predictions(preds_csv, truth_csv)
+        for key, val in metrics.items():
+            print(f"{key}: {val}")
+
+
+if __name__ == "__main__":
+    main()

code/tests/test_evaluation.py

@@ -0,0 +1,81 @@
+"""Tests for evaluation module."""
+import csv
+import pytest
+from evaluation.main import evaluate_predictions
+
+
+def test_evaluate_predictions_missing_files(tmp_path):
+    """Test evaluation logic when files are missing."""
+    # Pass non-existent paths
+    pred_path = tmp_path / "preds.csv"
+    truth_path = tmp_path / "truth.csv"
+
+    metrics = evaluate_predictions(pred_path, truth_path)
+    assert not metrics
+
+
+def test_evaluate_predictions_empty_files(tmp_path):
+    """Test evaluation logic when files are empty."""
+    pred_path = tmp_path / "preds.csv"
+    truth_path = tmp_path / "truth.csv"
+
+    # Create empty CSVs with just headers
+    headers = [
+        "user_id", "claim_status", "issue_type", "object_part", "severity"
+    ]
+    for path in [pred_path, truth_path]:
+        with open(path, "w", newline="", encoding="utf-8") as f:
+            writer = csv.writer(f)
+            writer.writerow(headers)
+
+    metrics = evaluate_predictions(pred_path, truth_path)
+    assert metrics["total"] == 0
+    assert metrics["accuracy_claim_status"] == pytest.approx(0.0)
+
+
+def test_evaluate_predictions_accuracy(tmp_path):
+    """Test evaluation calculation logic."""
+    pred_path = tmp_path / "preds.csv"
+    truth_path = tmp_path / "truth.csv"
+
+    headers = [
+        "user_id", "claim_status", "issue_type", "object_part", "severity"
+    ]
+
+    truth_data = [
+        ["user_1", "supported", "dent", "door", "low"],
+        ["user_2", "contradicted", "scratch", "hood", "medium"],
+        ["user_3", "not_enough_information", "unknown", "unknown", "unknown"],
+    ]
+
+    pred_data = [
+        # All correct
+        ["user_1", "supported", "dent", "door", "low"],
+        # 1 correct
+        ["user_2", "supported", "scratch", "door", "high"],
+        # All correct
+        ["user_3", "not_enough_information", "unknown", "unknown", "unknown"],
+    ]
+
+    with open(truth_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.writer(f)
+        writer.writerow(headers)
+        writer.writerows(truth_data)
+
+    with open(pred_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.writer(f)
+        writer.writerow(headers)
+        writer.writerows(pred_data)
+
+    metrics = evaluate_predictions(pred_path, truth_path)
+
+    assert metrics["total"] == 3
+    assert metrics["correct_claim_status"] == 2
+    assert metrics["correct_issue_type"] == 3
+    assert metrics["correct_object_part"] == 2
+    assert metrics["correct_severity"] == 2
+
+    assert metrics["accuracy_claim_status"] == pytest.approx(2 / 3)
+    assert metrics["accuracy_issue_type"] == pytest.approx(1.0)
+    assert metrics["accuracy_object_part"] == pytest.approx(2 / 3)
+    assert metrics["accuracy_severity"] == pytest.approx(2 / 3)