From 7d225c4af364832b1d4d01d76f337f051e65e7eb Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Fri, 19 Jun 2026 08:38:14 +0000 Subject: [PATCH 1/3] test: add evaluation metrics testing and implementation Adds the evaluate_predictions function to compute accuracy metrics from CSV files. Includes test_evaluation.py with missing file, empty file, and accuracy correctness scenarios. Co-authored-by: NITISH-R-G <225521762+NITISH-R-G@users.noreply.github.com> --- code/evaluation/main.py | 49 +++++++++++++++++++++++++ code/tests/test_evaluation.py | 68 +++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 code/tests/test_evaluation.py diff --git a/code/evaluation/main.py b/code/evaluation/main.py index e69de29b..78744f24 100644 --- a/code/evaluation/main.py +++ b/code/evaluation/main.py @@ -0,0 +1,49 @@ +import csv +import json +from pathlib import Path + +def evaluate_predictions(predictions_path: Path, truth_path: Path) -> dict: + """Evaluate accuracy of predictions against ground truth.""" + if not predictions_path.exists() or not truth_path.exists(): + return {} + + with open(predictions_path, 'r', encoding='utf-8') as f: + preds = list(csv.DictReader(f)) + with open(truth_path, 'r', encoding='utf-8') as f: + truth = {row['user_id']: row for row in csv.DictReader(f)} + + metrics = { + 'total': len(preds), + 'correct_claim_status': 0, + 'correct_issue_type': 0, + 'correct_object_part': 0, + 'correct_severity': 0 + } + + for pred in preds: + user_id = pred.get('user_id') + if not user_id or user_id not in truth: + continue + + t = truth[user_id] + if pred.get('claim_status') == t.get('claim_status'): + metrics['correct_claim_status'] += 1 + if pred.get('issue_type') == t.get('issue_type'): + metrics['correct_issue_type'] += 1 + if pred.get('object_part') == t.get('object_part'): + metrics['correct_object_part'] += 1 + if pred.get('severity') == t.get('severity'): + metrics['correct_severity'] += 1 + + if metrics['total'] > 0: + metrics['accuracy_claim_status'] = metrics['correct_claim_status'] / metrics['total'] + metrics['accuracy_issue_type'] = metrics['correct_issue_type'] / metrics['total'] + metrics['accuracy_object_part'] = metrics['correct_object_part'] / metrics['total'] + metrics['accuracy_severity'] = metrics['correct_severity'] / metrics['total'] + else: + metrics['accuracy_claim_status'] = 0.0 + metrics['accuracy_issue_type'] = 0.0 + metrics['accuracy_object_part'] = 0.0 + metrics['accuracy_severity'] = 0.0 + + return metrics diff --git a/code/tests/test_evaluation.py b/code/tests/test_evaluation.py new file mode 100644 index 00000000..58b2010e --- /dev/null +++ b/code/tests/test_evaluation.py @@ -0,0 +1,68 @@ +import csv +from pathlib import Path +import pytest +from evaluation.main import evaluate_predictions + +def test_evaluate_predictions_missing_files(tmp_path): + # Pass non-existent paths + pred_path = tmp_path / "preds.csv" + truth_path = tmp_path / "truth.csv" + + metrics = evaluate_predictions(pred_path, truth_path) + assert metrics == {} + +def test_evaluate_predictions_empty_files(tmp_path): + pred_path = tmp_path / "preds.csv" + truth_path = tmp_path / "truth.csv" + + # Create empty CSVs with just headers + headers = ["user_id", "claim_status", "issue_type", "object_part", "severity"] + for path in [pred_path, truth_path]: + with open(path, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(headers) + + metrics = evaluate_predictions(pred_path, truth_path) + assert metrics["total"] == 0 + assert metrics["accuracy_claim_status"] == 0.0 + +def test_evaluate_predictions_accuracy(tmp_path): + pred_path = tmp_path / "preds.csv" + truth_path = tmp_path / "truth.csv" + + headers = ["user_id", "claim_status", "issue_type", "object_part", "severity"] + + truth_data = [ + ["user_1", "supported", "dent", "door", "low"], + ["user_2", "contradicted", "scratch", "hood", "medium"], + ["user_3", "not_enough_information", "unknown", "unknown", "unknown"], + ] + + pred_data = [ + ["user_1", "supported", "dent", "door", "low"], # All correct + ["user_2", "supported", "scratch", "door", "high"], # 1 correct + ["user_3", "not_enough_information", "unknown", "unknown", "unknown"], # All correct + ] + + with open(truth_path, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(headers) + writer.writerows(truth_data) + + with open(pred_path, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(headers) + writer.writerows(pred_data) + + metrics = evaluate_predictions(pred_path, truth_path) + + assert metrics["total"] == 3 + assert metrics["correct_claim_status"] == 2 + assert metrics["correct_issue_type"] == 3 + assert metrics["correct_object_part"] == 2 + assert metrics["correct_severity"] == 2 + + assert metrics["accuracy_claim_status"] == 2/3 + assert metrics["accuracy_issue_type"] == 3/3 + assert metrics["accuracy_object_part"] == 2/3 + assert metrics["accuracy_severity"] == 2/3 From 8f2979bf84cfee328673dcf3c410b3e779299988 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Fri, 19 Jun 2026 08:42:19 +0000 Subject: [PATCH 2/3] fix: resolve linter errors in test evaluation Resolves flake8 and pylint errors found in SonarCloud Code Analysis. Removes trailing whitespaces, unused imports, line too long errors, and adds missing docstrings for code/evaluation/main.py and code/tests/test_evaluation.py. Also fixes missing encoding in file opening calls. Co-authored-by: NITISH-R-G <225521762+NITISH-R-G@users.noreply.github.com> --- code/evaluation/main.py | 18 ++++++++++----- code/tests/test_evaluation.py | 42 ++++++++++++++++++++++------------- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/code/evaluation/main.py b/code/evaluation/main.py index 78744f24..55214da5 100644 --- a/code/evaluation/main.py +++ b/code/evaluation/main.py @@ -1,7 +1,8 @@ +"""Metrics evaluation module.""" import csv -import json from pathlib import Path + def evaluate_predictions(predictions_path: Path, truth_path: Path) -> dict: """Evaluate accuracy of predictions against ground truth.""" if not predictions_path.exists() or not truth_path.exists(): @@ -35,11 +36,16 @@ def evaluate_predictions(predictions_path: Path, truth_path: Path) -> dict: if pred.get('severity') == t.get('severity'): metrics['correct_severity'] += 1 - if metrics['total'] > 0: - metrics['accuracy_claim_status'] = metrics['correct_claim_status'] / metrics['total'] - metrics['accuracy_issue_type'] = metrics['correct_issue_type'] / metrics['total'] - metrics['accuracy_object_part'] = metrics['correct_object_part'] / metrics['total'] - metrics['accuracy_severity'] = metrics['correct_severity'] / metrics['total'] + total = metrics['total'] + if total > 0: + metrics['accuracy_claim_status'] = ( + metrics['correct_claim_status'] / total + ) + metrics['accuracy_issue_type'] = metrics['correct_issue_type'] / total + metrics['accuracy_object_part'] = ( + metrics['correct_object_part'] / total + ) + metrics['accuracy_severity'] = metrics['correct_severity'] / total else: metrics['accuracy_claim_status'] = 0.0 metrics['accuracy_issue_type'] = 0.0 diff --git a/code/tests/test_evaluation.py b/code/tests/test_evaluation.py index 58b2010e..fb41fd17 100644 --- a/code/tests/test_evaluation.py +++ b/code/tests/test_evaluation.py @@ -1,24 +1,29 @@ +"""Tests for evaluation module.""" import csv -from pathlib import Path -import pytest from evaluation.main import evaluate_predictions + def test_evaluate_predictions_missing_files(tmp_path): + """Test evaluation logic when files are missing.""" # Pass non-existent paths pred_path = tmp_path / "preds.csv" truth_path = tmp_path / "truth.csv" metrics = evaluate_predictions(pred_path, truth_path) - assert metrics == {} + assert not metrics + def test_evaluate_predictions_empty_files(tmp_path): + """Test evaluation logic when files are empty.""" pred_path = tmp_path / "preds.csv" truth_path = tmp_path / "truth.csv" # Create empty CSVs with just headers - headers = ["user_id", "claim_status", "issue_type", "object_part", "severity"] + headers = [ + "user_id", "claim_status", "issue_type", "object_part", "severity" + ] for path in [pred_path, truth_path]: - with open(path, "w", newline="") as f: + with open(path, "w", newline="", encoding="utf-8") as f: writer = csv.writer(f) writer.writerow(headers) @@ -26,11 +31,15 @@ def test_evaluate_predictions_empty_files(tmp_path): assert metrics["total"] == 0 assert metrics["accuracy_claim_status"] == 0.0 + def test_evaluate_predictions_accuracy(tmp_path): + """Test evaluation calculation logic.""" pred_path = tmp_path / "preds.csv" truth_path = tmp_path / "truth.csv" - headers = ["user_id", "claim_status", "issue_type", "object_part", "severity"] + headers = [ + "user_id", "claim_status", "issue_type", "object_part", "severity" + ] truth_data = [ ["user_1", "supported", "dent", "door", "low"], @@ -39,17 +48,20 @@ def test_evaluate_predictions_accuracy(tmp_path): ] pred_data = [ - ["user_1", "supported", "dent", "door", "low"], # All correct - ["user_2", "supported", "scratch", "door", "high"], # 1 correct - ["user_3", "not_enough_information", "unknown", "unknown", "unknown"], # All correct + # All correct + ["user_1", "supported", "dent", "door", "low"], + # 1 correct + ["user_2", "supported", "scratch", "door", "high"], + # All correct + ["user_3", "not_enough_information", "unknown", "unknown", "unknown"], ] - with open(truth_path, "w", newline="") as f: + with open(truth_path, "w", newline="", encoding="utf-8") as f: writer = csv.writer(f) writer.writerow(headers) writer.writerows(truth_data) - with open(pred_path, "w", newline="") as f: + with open(pred_path, "w", newline="", encoding="utf-8") as f: writer = csv.writer(f) writer.writerow(headers) writer.writerows(pred_data) @@ -62,7 +74,7 @@ def test_evaluate_predictions_accuracy(tmp_path): assert metrics["correct_object_part"] == 2 assert metrics["correct_severity"] == 2 - assert metrics["accuracy_claim_status"] == 2/3 - assert metrics["accuracy_issue_type"] == 3/3 - assert metrics["accuracy_object_part"] == 2/3 - assert metrics["accuracy_severity"] == 2/3 + assert metrics["accuracy_claim_status"] == 2 / 3 + assert metrics["accuracy_issue_type"] == 3 / 3 + assert metrics["accuracy_object_part"] == 2 / 3 + assert metrics["accuracy_severity"] == 2 / 3 From 49f0a12deb4e48a4ff625fcf561ab6ca2a121832 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Fri, 19 Jun 2026 08:46:06 +0000 Subject: [PATCH 3/3] refactor: improve evaluate_predictions typing and test assertions Enhances `evaluate_predictions` in `code/evaluation/main.py` with proper type hints (`Dict[str, Union[int, float]]`), removes duplicate code by using a field loop for calculating metrics, and adds a `main` execution block for local script runs. Refactors `code/tests/test_evaluation.py` to use `pytest.approx` for floating point assertions to avoid precision issues reported by SonarCloud. Co-authored-by: NITISH-R-G <225521762+NITISH-R-G@users.noreply.github.com> --- code/evaluation/main.py | 60 +++++++++++++++++++---------------- code/tests/test_evaluation.py | 11 ++++--- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/code/evaluation/main.py b/code/evaluation/main.py index 55214da5..ca96fb67 100644 --- a/code/evaluation/main.py +++ b/code/evaluation/main.py @@ -1,9 +1,12 @@ """Metrics evaluation module.""" import csv from pathlib import Path +from typing import Dict, Union -def evaluate_predictions(predictions_path: Path, truth_path: Path) -> dict: +def evaluate_predictions( + predictions_path: Path, truth_path: Path +) -> Dict[str, Union[int, float]]: """Evaluate accuracy of predictions against ground truth.""" if not predictions_path.exists() or not truth_path.exists(): return {} @@ -13,7 +16,7 @@ def evaluate_predictions(predictions_path: Path, truth_path: Path) -> dict: with open(truth_path, 'r', encoding='utf-8') as f: truth = {row['user_id']: row for row in csv.DictReader(f)} - metrics = { + metrics: Dict[str, Union[int, float]] = { 'total': len(preds), 'correct_claim_status': 0, 'correct_issue_type': 0, @@ -21,35 +24,38 @@ def evaluate_predictions(predictions_path: Path, truth_path: Path) -> dict: 'correct_severity': 0 } + fields = ['claim_status', 'issue_type', 'object_part', 'severity'] + for pred in preds: user_id = pred.get('user_id') if not user_id or user_id not in truth: continue - t = truth[user_id] - if pred.get('claim_status') == t.get('claim_status'): - metrics['correct_claim_status'] += 1 - if pred.get('issue_type') == t.get('issue_type'): - metrics['correct_issue_type'] += 1 - if pred.get('object_part') == t.get('object_part'): - metrics['correct_object_part'] += 1 - if pred.get('severity') == t.get('severity'): - metrics['correct_severity'] += 1 - - total = metrics['total'] - if total > 0: - metrics['accuracy_claim_status'] = ( - metrics['correct_claim_status'] / total - ) - metrics['accuracy_issue_type'] = metrics['correct_issue_type'] / total - metrics['accuracy_object_part'] = ( - metrics['correct_object_part'] / total - ) - metrics['accuracy_severity'] = metrics['correct_severity'] / total - else: - metrics['accuracy_claim_status'] = 0.0 - metrics['accuracy_issue_type'] = 0.0 - metrics['accuracy_object_part'] = 0.0 - metrics['accuracy_severity'] = 0.0 + t_row = truth[user_id] + for field in fields: + if pred.get(field) == t_row.get(field): + metrics[f'correct_{field}'] += 1 # type: ignore + + total = int(metrics['total']) + for field in fields: + correct = int(metrics[f'correct_{field}']) + acc_key = f'accuracy_{field}' + metrics[acc_key] = correct / total if total > 0 else 0.0 return metrics + + +def main() -> None: + """Run evaluation on sample claims.""" + base_dir = Path(__file__).parent.parent.parent + preds_csv = base_dir / "output.csv" + truth_csv = base_dir / "dataset" / "sample_claims.csv" + + if preds_csv.exists() and truth_csv.exists(): + metrics = evaluate_predictions(preds_csv, truth_csv) + for key, val in metrics.items(): + print(f"{key}: {val}") + + +if __name__ == "__main__": + main() diff --git a/code/tests/test_evaluation.py b/code/tests/test_evaluation.py index fb41fd17..64638cd9 100644 --- a/code/tests/test_evaluation.py +++ b/code/tests/test_evaluation.py @@ -1,5 +1,6 @@ """Tests for evaluation module.""" import csv +import pytest from evaluation.main import evaluate_predictions @@ -29,7 +30,7 @@ def test_evaluate_predictions_empty_files(tmp_path): metrics = evaluate_predictions(pred_path, truth_path) assert metrics["total"] == 0 - assert metrics["accuracy_claim_status"] == 0.0 + assert metrics["accuracy_claim_status"] == pytest.approx(0.0) def test_evaluate_predictions_accuracy(tmp_path): @@ -74,7 +75,7 @@ def test_evaluate_predictions_accuracy(tmp_path): assert metrics["correct_object_part"] == 2 assert metrics["correct_severity"] == 2 - assert metrics["accuracy_claim_status"] == 2 / 3 - assert metrics["accuracy_issue_type"] == 3 / 3 - assert metrics["accuracy_object_part"] == 2 / 3 - assert metrics["accuracy_severity"] == 2 / 3 + assert metrics["accuracy_claim_status"] == pytest.approx(2 / 3) + assert metrics["accuracy_issue_type"] == pytest.approx(1.0) + assert metrics["accuracy_object_part"] == pytest.approx(2 / 3) + assert metrics["accuracy_severity"] == pytest.approx(2 / 3)