Merge branch 'feat/feedback-loop-for-unmatched-test-results' of github.com:codeflash-ai/codeflash into feat/feedback-loop-for-unmatched-test-results

Author: mohammedahmed18

codeflash/code_utils/code_utils.py

@@ -362,6 +362,10 @@ def exit_with_message(message: str, *, error_on_exit: bool = False) -> None:
     sys.exit(1 if error_on_exit else 0)
 
 
+def shorten_pytest_error(pytest_error_string: str) -> str:
+    return "\n".join(re.findall(r"^[E>] +(.*)$", pytest_error_string, re.MULTILINE))
+
+
 def extract_unique_errors(pytest_output: str) -> set[str]:
     unique_errors = set()
 

codeflash/verification/equivalence.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
 
+import reprlib
 import sys
 from typing import TYPE_CHECKING
 
 from codeflash.cli_cmds.console import logger
+from codeflash.code_utils.code_utils import shorten_pytest_error
 from codeflash.models.models import TestDiff, TestDiffScope, TestResults, TestType, VerificationType
 from codeflash.verification.comparator import comparator
 
@@ -57,11 +59,29 @@ def compare_test_results(original_results: TestResults, candidate_results: TestR
             if candidate_test_failures
             else ""
         )
+        if cdd_pytest_error:
+            cdd_pytest_error = shorten_pytest_error(cdd_pytest_error)
         original_pytest_error = (
             original_test_failures.get(original_test_result.id.test_fn_qualified_name(), "")
             if original_test_failures
             else ""
         )
+        if original_pytest_error:
+            original_pytest_error = shorten_pytest_error(original_pytest_error)
+        test_src_code = original_test_result.id.get_src_code(original_test_result.file_name)
+        test_diff = TestDiff(
+            scope=TestDiffScope.RETURN_VALUE,
+            original_value=reprlib.repr(original_test_result.return_value),
+            candidate_value=reprlib.repr(cdd_test_result.return_value),
+            test_src_code=test_src_code,
+            candidate_pytest_error=cdd_pytest_error,
+            original_pass=original_test_result.did_pass,
+            candidate_pass=cdd_test_result.did_pass,
+            original_pytest_error=original_pytest_error,
+        )
+        if not comparator(original_test_result.return_value, cdd_test_result.return_value, superset_obj=superset_obj):
+            test_diff.scope = TestDiffScope.RETURN_VALUE
+            test_diffs.append(test_diff)
 
         if not comparator(original_test_result.return_value, cdd_test_result.return_value, superset_obj=superset_obj):
             test_diffs.append(