diff --git a/code/evaluation/main.py b/code/evaluation/main.py index e69de29b..ca96fb67 100644 --- a/code/evaluation/main.py +++ b/code/evaluation/main.py @@ -0,0 +1,61 @@ +"""Metrics evaluation module.""" +import csv +from pathlib import Path +from typing import Dict, Union + + +def evaluate_predictions( + predictions_path: Path, truth_path: Path +) -> Dict[str, Union[int, float]]: + """Evaluate accuracy of predictions against ground truth.""" + if not predictions_path.exists() or not truth_path.exists(): + return {} + + with open(predictions_path, 'r', encoding='utf-8') as f: + preds = list(csv.DictReader(f)) + with open(truth_path, 'r', encoding='utf-8') as f: + truth = {row['user_id']: row for row in csv.DictReader(f)} + + metrics: Dict[str, Union[int, float]] = { + 'total': len(preds), + 'correct_claim_status': 0, + 'correct_issue_type': 0, + 'correct_object_part': 0, + 'correct_severity': 0 + } + + fields = ['claim_status', 'issue_type', 'object_part', 'severity'] + + for pred in preds: + user_id = pred.get('user_id') + if not user_id or user_id not in truth: + continue + + t_row = truth[user_id] + for field in fields: + if pred.get(field) == t_row.get(field): + metrics[f'correct_{field}'] += 1 # type: ignore + + total = int(metrics['total']) + for field in fields: + correct = int(metrics[f'correct_{field}']) + acc_key = f'accuracy_{field}' + metrics[acc_key] = correct / total if total > 0 else 0.0 + + return metrics + + +def main() -> None: + """Run evaluation on sample claims.""" + base_dir = Path(__file__).parent.parent.parent + preds_csv = base_dir / "output.csv" + truth_csv = base_dir / "dataset" / "sample_claims.csv" + + if preds_csv.exists() and truth_csv.exists(): + metrics = evaluate_predictions(preds_csv, truth_csv) + for key, val in metrics.items(): + print(f"{key}: {val}") + + +if __name__ == "__main__": + main() diff --git a/code/tests/test_evaluation.py b/code/tests/test_evaluation.py new file mode 100644 index 00000000..64638cd9 --- /dev/null +++ b/code/tests/test_evaluation.py @@ -0,0 +1,81 @@ +"""Tests for evaluation module.""" +import csv +import pytest +from evaluation.main import evaluate_predictions + + +def test_evaluate_predictions_missing_files(tmp_path): + """Test evaluation logic when files are missing.""" + # Pass non-existent paths + pred_path = tmp_path / "preds.csv" + truth_path = tmp_path / "truth.csv" + + metrics = evaluate_predictions(pred_path, truth_path) + assert not metrics + + +def test_evaluate_predictions_empty_files(tmp_path): + """Test evaluation logic when files are empty.""" + pred_path = tmp_path / "preds.csv" + truth_path = tmp_path / "truth.csv" + + # Create empty CSVs with just headers + headers = [ + "user_id", "claim_status", "issue_type", "object_part", "severity" + ] + for path in [pred_path, truth_path]: + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(headers) + + metrics = evaluate_predictions(pred_path, truth_path) + assert metrics["total"] == 0 + assert metrics["accuracy_claim_status"] == pytest.approx(0.0) + + +def test_evaluate_predictions_accuracy(tmp_path): + """Test evaluation calculation logic.""" + pred_path = tmp_path / "preds.csv" + truth_path = tmp_path / "truth.csv" + + headers = [ + "user_id", "claim_status", "issue_type", "object_part", "severity" + ] + + truth_data = [ + ["user_1", "supported", "dent", "door", "low"], + ["user_2", "contradicted", "scratch", "hood", "medium"], + ["user_3", "not_enough_information", "unknown", "unknown", "unknown"], + ] + + pred_data = [ + # All correct + ["user_1", "supported", "dent", "door", "low"], + # 1 correct + ["user_2", "supported", "scratch", "door", "high"], + # All correct + ["user_3", "not_enough_information", "unknown", "unknown", "unknown"], + ] + + with open(truth_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(headers) + writer.writerows(truth_data) + + with open(pred_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(headers) + writer.writerows(pred_data) + + metrics = evaluate_predictions(pred_path, truth_path) + + assert metrics["total"] == 3 + assert metrics["correct_claim_status"] == 2 + assert metrics["correct_issue_type"] == 3 + assert metrics["correct_object_part"] == 2 + assert metrics["correct_severity"] == 2 + + assert metrics["accuracy_claim_status"] == pytest.approx(2 / 3) + assert metrics["accuracy_issue_type"] == pytest.approx(1.0) + assert metrics["accuracy_object_part"] == pytest.approx(2 / 3) + assert metrics["accuracy_severity"] == pytest.approx(2 / 3)