From e4cbf954261da0b0142fc64778fe85c6d92b464c Mon Sep 17 00:00:00 2001 From: ali Date: Mon, 8 Sep 2025 15:18:40 +0300 Subject: [PATCH 01/10] exp --- codeflash/code_utils/config_consts.py | 2 ++ codeflash/verification/pytest_plugin.py | 24 +++++++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/codeflash/code_utils/config_consts.py b/codeflash/code_utils/config_consts.py index 50b4bce16..cdfdbe2d8 100644 --- a/codeflash/code_utils/config_consts.py +++ b/codeflash/code_utils/config_consts.py @@ -7,6 +7,8 @@ MAX_CUMULATIVE_TEST_RUNTIME_NANOSECONDS = 100e6 # 100ms N_TESTS_TO_GENERATE = 2 TOTAL_LOOPING_TIME = 10.0 # 10 second candidate benchmarking budget +CONSISTENT_LOOP_COUNT = 5 +CONSISTENT_DURATION_TOLERANCE = 0.05 COVERAGE_THRESHOLD = 60.0 MIN_TESTCASE_PASSED_THRESHOLD = 6 REPEAT_OPTIMIZATION_PROBABILITY = 0.1 diff --git a/codeflash/verification/pytest_plugin.py b/codeflash/verification/pytest_plugin.py index 85cd4d13c..4e9d2d7cd 100644 --- a/codeflash/verification/pytest_plugin.py +++ b/codeflash/verification/pytest_plugin.py @@ -19,6 +19,8 @@ import pytest from pluggy import HookspecMarker +from codeflash.code_utils.config_consts import CONSISTENT_DURATION_TOLERANCE, CONSISTENT_LOOP_COUNT + if TYPE_CHECKING: from _pytest.config import Config, Parser from _pytest.main import Session @@ -271,6 +273,8 @@ def __init__(self, config: Config) -> None: @hookspec(firstresult=True) def pytest_runtestloop(self, session: Session) -> bool: + durations: list[float] = [] + """Reimplement the test loop but loop for the user defined amount of time.""" if session.testsfailed and not session.config.option.continue_on_collection_errors: msg = "{} error{} during collection".format(session.testsfailed, "s" if session.testsfailed != 1 else "") @@ -284,9 +288,9 @@ def pytest_runtestloop(self, session: Session) -> bool: count: int = 0 - while total_time >= SHORTEST_AMOUNT_OF_TIME: # need to run at least one for normal tests + while total_time >= SHORTEST_AMOUNT_OF_TIME: count += 1 - total_time = self._get_total_time(session) + loop_start = _ORIGINAL_TIME_TIME() for index, item in enumerate(session.items): item: pytest.Item = item # noqa: PLW0127, PLW2901 @@ -304,8 +308,22 @@ def pytest_runtestloop(self, session: Session) -> bool: raise session.Failed(session.shouldfail) if session.shouldstop: raise session.Interrupted(session.shouldstop) + + loop_end = _ORIGINAL_TIME_TIME() + loop_duration = loop_end - loop_start + durations.append(loop_duration) + + # Consistency check + if len(durations) >= CONSISTENT_LOOP_COUNT: + recent = durations[-CONSISTENT_LOOP_COUNT:] + avg = sum(recent) / CONSISTENT_LOOP_COUNT + consistent = all(abs(d - avg) / avg <= CONSISTENT_DURATION_TOLERANCE for d in recent) + if consistent: + break + if self._timed_out(session, start_time, count): - break # exit loop + break + _ORIGINAL_TIME_SLEEP(self._get_delay_time(session)) return True From 01ad6261ad7c790ab9e67f8a08a061ce61cec054 Mon Sep 17 00:00:00 2001 From: ali Date: Tue, 9 Sep 2025 16:31:52 +0300 Subject: [PATCH 02/10] still experimenting --- code_to_optimize/find_common_tags.py | 3 + codeflash/code_utils/line_profile_utils.py | 88 ++-- codeflash/dummy.py | 15 + codeflash/dummy2.py | 78 ++++ .../parse_line_profile_test_output.py | 3 +- out.json | 439 ++++++++++++++++++ out.lprof | 26 ++ tests/test_instrument_line_profiler.py | 10 +- 8 files changed, 624 insertions(+), 38 deletions(-) create mode 100644 codeflash/dummy.py create mode 100644 codeflash/dummy2.py create mode 100644 out.json create mode 100644 out.lprof diff --git a/code_to_optimize/find_common_tags.py b/code_to_optimize/find_common_tags.py index 016905bfc..803ada4f0 100644 --- a/code_to_optimize/find_common_tags.py +++ b/code_to_optimize/find_common_tags.py @@ -1,6 +1,9 @@ from __future__ import annotations +from codeflash.code_utils.line_profile_utils import LineProfilerDecorator + +@codeflash_line_profile def find_common_tags(articles: list[dict[str, list[str]]]) -> set[str]: if not articles: return set() diff --git a/codeflash/code_utils/line_profile_utils.py b/codeflash/code_utils/line_profile_utils.py index 498571578..11ea3dea2 100644 --- a/codeflash/code_utils/line_profile_utils.py +++ b/codeflash/code_utils/line_profile_utils.py @@ -2,12 +2,14 @@ from __future__ import annotations +import functools from collections import defaultdict from pathlib import Path -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Any, Callable, Union import isort import libcst as cst +from line_profiler import LineProfiler from codeflash.code_utils.code_utils import get_run_tmp_file @@ -45,6 +47,7 @@ def leave_ClassDef(self, original_node: cst.ClassDef, updated_node: cst.ClassDef self.context_stack.pop() return updated_node + # f6fa22dd-d711-425b-bed4-d2844ad6af36 def visit_FunctionDef(self, node: cst.FunctionDef) -> None: # Track when we enter a function self.context_stack.append(node.name.value) @@ -78,6 +81,17 @@ def _is_target_decorator(self, decorator_node: Union[cst.Name, cst.Attribute, cs return False +# Helper to get full module path as string +def get_module_path(node: Union[cst.Name, cst.Attribute]) -> str: + parts = [] + while isinstance(node, cst.Attribute): + parts.insert(0, node.attr.value) + node = node.value + if isinstance(node, cst.Name): + parts.insert(0, node.value) + return ".".join(parts) + + class ProfileEnableTransformer(cst.CSTTransformer): def __init__(self, filename: str) -> None: # Flag to track if we found the import statement @@ -87,21 +101,13 @@ def __init__(self, filename: str) -> None: self.filename = filename def leave_ImportFrom(self, original_node: cst.ImportFrom, updated_node: cst.ImportFrom) -> cst.ImportFrom: - # Check if this is the line profiler import statement - if ( - isinstance(original_node.module, cst.Name) - and original_node.module.value == "line_profiler" - and any( - name.name.value == "profile" and (not name.asname or name.asname.name.value == "codeflash_line_profile") - for name in original_node.names - ) + module_path = get_module_path(original_node.module) + + if module_path == "codeflash.code_utils.line_profile_utils" and any( + name.name.value == "LineProfilerDecorator" for name in original_node.names ): + print("Found import statement") self.found_import = True - # Get the indentation from the original node - if hasattr(original_node, "leading_lines"): - leading_whitespace = original_node.leading_lines[-1].whitespace if original_node.leading_lines else "" - self.import_indentation = leading_whitespace - return updated_node def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> cst.Module: # noqa: ARG002 @@ -116,26 +122,19 @@ def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> c for i, stmt in enumerate(new_body): if isinstance(stmt, cst.SimpleStatementLine): for small_stmt in stmt.body: - if isinstance(small_stmt, cst.ImportFrom) and ( - isinstance(small_stmt.module, cst.Name) - and small_stmt.module.value == "line_profiler" - and any( - name.name.value == "profile" - and (not name.asname or name.asname.name.value == "codeflash_line_profile") - for name in small_stmt.names - ) - ): - import_index = i - break + if isinstance(small_stmt, cst.ImportFrom): + module_path = get_module_path(small_stmt.module) + if module_path == "codeflash.code_utils.line_profile_utils" and any( + name.name.value == "LineProfilerDecorator" for name in small_stmt.names + ): + import_index = i + break if import_index is not None: break if import_index is not None: - # Create the new enable statement to insert after the import - enable_statement = cst.parse_statement(f"codeflash_line_profile.enable(output_prefix='{self.filename}')") - - # Insert the new statement after the import statement - new_body.insert(import_index + 1, enable_statement) + assignment = cst.parse_statement(f"codeflash_line_profile = LineProfilerDecorator('{self.filename}')") + new_body.insert(import_index + 1, assignment) # Create a new module with the updated body return updated_node.with_changes(body=new_body) @@ -196,7 +195,7 @@ def add_decorator_imports(function_to_optimize: FunctionToOptimize, code_context # self.function_to_optimize, file_path_to_helper_classes, self.test_cfg.tests_root # grouped iteration, file to fns to optimize, from line_profiler import profile as codeflash_line_profile file_paths = defaultdict(list) - line_profile_output_file = get_run_tmp_file(Path("baseline_lprof")) + line_profile_output_file = get_run_tmp_file(Path("baseline_lprof.txt")) file_paths[function_to_optimize.file_path].append(function_to_optimize.qualified_name) for elem in code_context.helper_functions: file_paths[elem.file_path].append(elem.qualified_name) @@ -210,7 +209,7 @@ def add_decorator_imports(function_to_optimize: FunctionToOptimize, code_context module_node = add_decorator_to_qualified_function(module_node, fn_name, "codeflash_line_profile") # add imports # Create a transformer to add the import - transformer = ImportAdder("from line_profiler import profile as codeflash_line_profile") + transformer = ImportAdder("from codeflash.code_utils.line_profile_utils import LineProfilerDecorator") # Apply the transformer to add the import module_node = module_node.visit(transformer) modified_code = isort.code(module_node.code, float_to_top=True) @@ -222,3 +221,28 @@ def add_decorator_imports(function_to_optimize: FunctionToOptimize, code_context modified_code = add_profile_enable(file_contents, str(line_profile_output_file)) function_to_optimize.file_path.write_text(modified_code, "utf-8") return line_profile_output_file + + +class LineProfilerDecorator: + """Decorator class that profiles multiple functions and saves stats automatically.""" + + def __init__(self, output_file: str | Path) -> None: + self.output_file: Path = Path(output_file) + self.profiler: LineProfiler = LineProfiler() + # Ensure parent folder exists + self.output_file.parent.mkdir(parents=True, exist_ok=True) + + def __call__(self, func: Callable[..., Any]) -> Callable[..., Any]: + """Decorate a function to profile it and save stats after execution.""" + self.profiler.add_function(func) + + @functools.wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: # noqa: ANN401 + try: + return self.profiler(func)(*args, **kwargs) + finally: + # Save stats after each call + with self.output_file.open("w") as f: + self.profiler.print_stats(f) + + return wrapper diff --git a/codeflash/dummy.py b/codeflash/dummy.py new file mode 100644 index 000000000..b9468e19c --- /dev/null +++ b/codeflash/dummy.py @@ -0,0 +1,15 @@ +# @profile +# def my_function_to_profile(): +# # Your function's code here +# result = 0 +# for i in range(1000000): +# result += i * 2 +# return result +# +# +# def another_function_we_dont_care_about(): +# pass +# +# +# if __name__ == "__main__": +# my_function_to_profile() diff --git a/codeflash/dummy2.py b/codeflash/dummy2.py new file mode 100644 index 000000000..9f6d8163c --- /dev/null +++ b/codeflash/dummy2.py @@ -0,0 +1,78 @@ +# import random +# import time + +# from line_profiler import LineProfiler + +# codeflash_line_profile = LineProfiler() + + +# @codeflash_line_profile +# def complex_function(n: int) -> float: +# """A function with nested loops, math operations, and simulated I/O.""" + +# def helper_compute(x: float) -> float: +# """Nested helper doing some CPU-heavy math.""" +# total = 0.0 +# for i in range(1, 5000): +# total += math.sin(x * i) ** 2 + math.cos(x / (i + 1)) ** 3 +# return total + +# def helper_io_simulation(): +# """Simulate I/O-bound operations with tiny sleeps.""" +# for _ in range(5): +# time.sleep(0.001 * random.random()) # sleep for 0-1ms + +# result = 0.0 +# for i in range(1, n + 1): +# val = helper_compute(i) +# helper_io_simulation() +# result += val / (i + 1) + +# # some post-processing +# result = math.sqrt(result) + math.log1p(result) +# return result + +# complex_function(10) +# with open("foo.lprof", "w") as f: +# codeflash_line_profile.print_stats(f) + +# import functools +# from pathlib import Path + +# from line_profiler import LineProfiler + + +# class LineProfilerDecorator: +# """Decorator class that profiles multiple functions and saves stats automatically.""" +# def __init__(self, output_file: str | Path): +# self.output_file = Path(output_file) +# self.profiler = LineProfiler() +# # Ensure parent folder exists +# self.output_file.parent.mkdir(parents=True, exist_ok=True) + +# def __call__(self, func): +# """Decorate a function to profile it and save stats after execution.""" +# self.profiler.add_function(func) + +# @functools.wraps(func) +# def wrapper(*args, **kwargs): +# try: +# return self.profiler(func)(*args, **kwargs) +# finally: +# # Save stats after each call +# with self.output_file.open("w") as f: +# self.profiler.print_stats(f) + +# return wrapper + +# codeflash_line_profile = LineProfilerDecorator("baseline_lprof") + +# @codeflash_line_profile +# def foo(): +# for i in range(100): +# print("hello world") + +# if __name__ == "__main__": +# foo() + + diff --git a/codeflash/verification/parse_line_profile_test_output.py b/codeflash/verification/parse_line_profile_test_output.py index 1877c0654..177955c81 100644 --- a/codeflash/verification/parse_line_profile_test_output.py +++ b/codeflash/verification/parse_line_profile_test_output.py @@ -9,6 +9,7 @@ import dill as pickle +from codeflash.cli_cmds.console import logger from codeflash.code_utils.tabulate import tabulate if TYPE_CHECKING: @@ -78,9 +79,9 @@ def show_text(stats: dict) -> str: def parse_line_profile_results(line_profiler_output_file: Optional[Path]) -> dict: - line_profiler_output_file = line_profiler_output_file.with_suffix(".lprof") stats_dict = {} if not line_profiler_output_file.exists(): + logger.warning(f"Line profiler output file {line_profiler_output_file} does not exist.") return {"timings": {}, "unit": 0, "str_out": ""}, None with line_profiler_output_file.open("rb") as f: stats = pickle.load(f) diff --git a/out.json b/out.json new file mode 100644 index 000000000..b8749e2a3 --- /dev/null +++ b/out.json @@ -0,0 +1,439 @@ +{ + "alloc_samples": 0, + "args": [ + "--reduce-profile", + "python", + "codeflash/dummy.py" + ], + "elapsed_time_sec": 0.1318800449371338, + "entrypoint_dir": "/home/mohammed/Work/codeflash/codeflash", + "filename": "/home/mohammed/Work/codeflash/codeflash", + "files": { + "/home/mohammed/Work/codeflash/codeflash/dummy.py": { + "functions": [ + { + "cpu_samples_list": [], + "line": "my_function_to_profile", + "lineno": 3, + "memory_samples": [], + "n_avg_mb": 0.0, + "n_copy_mb_s": 0.0, + "n_core_utilization": 0.12357065668313703, + "n_cpu_percent_c": 11.008220197077355, + "n_cpu_percent_python": 87.84830514943228, + "n_gpu_avg_memory_mb": 0.0, + "n_gpu_peak_memory_mb": 0.0, + "n_gpu_percent": 0.0, + "n_growth_mb": 0.0, + "n_malloc_mb": 0.0, + "n_mallocs": 0, + "n_peak_mb": 0.0, + "n_python_fraction": 0, + "n_sys_percent": 1.1434746534903728, + "n_usage_fraction": 0 + } + ], + "imports": [], + "leaks": {}, + "lines": [ + { + "cpu_samples_list": [], + "end_outermost_loop": 1, + "end_region_line": 1, + "line": "\n", + "lineno": 1, + "memory_samples": [], + "n_avg_mb": 0, + "n_copy_mb_s": 0, + "n_core_utilization": 0, + "n_cpu_percent_c": 0, + "n_cpu_percent_python": 0, + "n_gpu_avg_memory_mb": 0, + "n_gpu_peak_memory_mb": 0, + "n_gpu_percent": 0, + "n_growth_mb": 0, + "n_malloc_mb": 0, + "n_mallocs": 0, + "n_peak_mb": 0, + "n_python_fraction": 0, + "n_sys_percent": 0, + "n_usage_fraction": 0, + "start_outermost_loop": 1, + "start_region_line": 1 + }, + { + "cpu_samples_list": [], + "end_outermost_loop": 2, + "end_region_line": 2, + "line": "\n", + "lineno": 2, + "memory_samples": [], + "n_avg_mb": 0, + "n_copy_mb_s": 0, + "n_core_utilization": 0, + "n_cpu_percent_c": 0, + "n_cpu_percent_python": 0, + "n_gpu_avg_memory_mb": 0, + "n_gpu_peak_memory_mb": 0, + "n_gpu_percent": 0, + "n_growth_mb": 0, + "n_malloc_mb": 0, + "n_mallocs": 0, + "n_peak_mb": 0, + "n_python_fraction": 0, + "n_sys_percent": 0, + "n_usage_fraction": 0, + "start_outermost_loop": 2, + "start_region_line": 2 + }, + { + "cpu_samples_list": [], + "end_outermost_loop": 3, + "end_region_line": 3, + "line": "@profile\n", + "lineno": 3, + "memory_samples": [], + "n_avg_mb": 0.0, + "n_copy_mb_s": 0.0, + "n_core_utilization": 0.0, + "n_cpu_percent_c": 0.0, + "n_cpu_percent_python": 0.0, + "n_gpu_avg_memory_mb": 0.0, + "n_gpu_peak_memory_mb": 0.0, + "n_gpu_percent": 0, + "n_growth_mb": 0.0, + "n_malloc_mb": 0.0, + "n_mallocs": 0, + "n_peak_mb": 0.0, + "n_python_fraction": 0, + "n_sys_percent": 0.0, + "n_usage_fraction": 0, + "start_outermost_loop": 3, + "start_region_line": 3 + }, + { + "cpu_samples_list": [], + "end_outermost_loop": 9, + "end_region_line": 9, + "line": "def my_function_to_profile():\n", + "lineno": 4, + "memory_samples": [], + "n_avg_mb": 0.0, + "n_copy_mb_s": 0.0, + "n_core_utilization": 0.0, + "n_cpu_percent_c": 0.0, + "n_cpu_percent_python": 0.0, + "n_gpu_avg_memory_mb": 0.0, + "n_gpu_peak_memory_mb": 0.0, + "n_gpu_percent": 0, + "n_growth_mb": 0.0, + "n_malloc_mb": 0.0, + "n_mallocs": 0, + "n_peak_mb": 0.0, + "n_python_fraction": 0, + "n_sys_percent": 0.0, + "n_usage_fraction": 0, + "start_outermost_loop": 4, + "start_region_line": 4 + }, + { + "cpu_samples_list": [], + "end_outermost_loop": 9, + "end_region_line": 9, + "line": " # Your function's code here\n", + "lineno": 5, + "memory_samples": [], + "n_avg_mb": 0.0, + "n_copy_mb_s": 0.0, + "n_core_utilization": 0.0, + "n_cpu_percent_c": 0.0, + "n_cpu_percent_python": 0.0, + "n_gpu_avg_memory_mb": 0.0, + "n_gpu_peak_memory_mb": 0.0, + "n_gpu_percent": 0, + "n_growth_mb": 0.0, + "n_malloc_mb": 0.0, + "n_mallocs": 0, + "n_peak_mb": 0.0, + "n_python_fraction": 0, + "n_sys_percent": 0.0, + "n_usage_fraction": 0, + "start_outermost_loop": 4, + "start_region_line": 4 + }, + { + "cpu_samples_list": [], + "end_outermost_loop": 6, + "end_region_line": 9, + "line": " result = 0\n", + "lineno": 6, + "memory_samples": [], + "n_avg_mb": 0.0, + "n_copy_mb_s": 0.0, + "n_core_utilization": 0.0, + "n_cpu_percent_c": 0.0, + "n_cpu_percent_python": 0.0, + "n_gpu_avg_memory_mb": 0.0, + "n_gpu_peak_memory_mb": 0.0, + "n_gpu_percent": 0, + "n_growth_mb": 0.0, + "n_malloc_mb": 0.0, + "n_mallocs": 0, + "n_peak_mb": 0.0, + "n_python_fraction": 0, + "n_sys_percent": 0.0, + "n_usage_fraction": 0, + "start_outermost_loop": 6, + "start_region_line": 4 + }, + { + "cpu_samples_list": [], + "end_outermost_loop": 8, + "end_region_line": 8, + "line": " for i in range(1000000):\n", + "lineno": 7, + "memory_samples": [], + "n_avg_mb": 0.0, + "n_copy_mb_s": 0.0, + "n_core_utilization": 0.0, + "n_cpu_percent_c": 0.0, + "n_cpu_percent_python": 0.0, + "n_gpu_avg_memory_mb": 0.0, + "n_gpu_peak_memory_mb": 0.0, + "n_gpu_percent": 0, + "n_growth_mb": 0.0, + "n_malloc_mb": 0.0, + "n_mallocs": 0, + "n_peak_mb": 0.0, + "n_python_fraction": 0, + "n_sys_percent": 0.0, + "n_usage_fraction": 0, + "start_outermost_loop": 7, + "start_region_line": 7 + }, + { + "cpu_samples_list": [ + 8818.4491195, + 8818.468866728, + 8818.46997304, + 8818.480837068, + 8818.486153326, + 8818.493161905, + 8818.51569137 + ], + "end_outermost_loop": 8, + "end_region_line": 8, + "line": " result += i * 2\n", + "lineno": 8, + "memory_samples": [], + "n_avg_mb": 0.0, + "n_copy_mb_s": 0.0, + "n_core_utilization": 0.12357065668313703, + "n_cpu_percent_c": 11.008220197077355, + "n_cpu_percent_python": 87.84830514943228, + "n_gpu_avg_memory_mb": 0.0, + "n_gpu_peak_memory_mb": 0.0, + "n_gpu_percent": 0.0, + "n_growth_mb": 0.0, + "n_malloc_mb": 0.0, + "n_mallocs": 0, + "n_peak_mb": 0.0, + "n_python_fraction": 0, + "n_sys_percent": 1.1434746534903728, + "n_usage_fraction": 0, + "start_outermost_loop": 7, + "start_region_line": 7 + }, + { + "cpu_samples_list": [], + "end_outermost_loop": 9, + "end_region_line": 9, + "line": " return result\n", + "lineno": 9, + "memory_samples": [], + "n_avg_mb": 0.0, + "n_copy_mb_s": 0.0, + "n_core_utilization": 0.0, + "n_cpu_percent_c": 0.0, + "n_cpu_percent_python": 0.0, + "n_gpu_avg_memory_mb": 0.0, + "n_gpu_peak_memory_mb": 0.0, + "n_gpu_percent": 0, + "n_growth_mb": 0.0, + "n_malloc_mb": 0.0, + "n_mallocs": 0, + "n_peak_mb": 0.0, + "n_python_fraction": 0, + "n_sys_percent": 0.0, + "n_usage_fraction": 0, + "start_outermost_loop": 9, + "start_region_line": 4 + }, + { + "cpu_samples_list": [], + "end_outermost_loop": 10, + "end_region_line": 10, + "line": "\n", + "lineno": 10, + "memory_samples": [], + "n_avg_mb": 0, + "n_copy_mb_s": 0, + "n_core_utilization": 0, + "n_cpu_percent_c": 0, + "n_cpu_percent_python": 0, + "n_gpu_avg_memory_mb": 0, + "n_gpu_peak_memory_mb": 0, + "n_gpu_percent": 0, + "n_growth_mb": 0, + "n_malloc_mb": 0, + "n_mallocs": 0, + "n_peak_mb": 0, + "n_python_fraction": 0, + "n_sys_percent": 0, + "n_usage_fraction": 0, + "start_outermost_loop": 10, + "start_region_line": 10 + }, + { + "cpu_samples_list": [], + "end_outermost_loop": 12, + "end_region_line": 12, + "line": "def another_function_we_dont_care_about():\n", + "lineno": 11, + "memory_samples": [], + "n_avg_mb": 0, + "n_copy_mb_s": 0, + "n_core_utilization": 0, + "n_cpu_percent_c": 0, + "n_cpu_percent_python": 0, + "n_gpu_avg_memory_mb": 0, + "n_gpu_peak_memory_mb": 0, + "n_gpu_percent": 0, + "n_growth_mb": 0, + "n_malloc_mb": 0, + "n_mallocs": 0, + "n_peak_mb": 0, + "n_python_fraction": 0, + "n_sys_percent": 0, + "n_usage_fraction": 0, + "start_outermost_loop": 11, + "start_region_line": 11 + }, + { + "cpu_samples_list": [], + "end_outermost_loop": 12, + "end_region_line": 12, + "line": " pass\n", + "lineno": 12, + "memory_samples": [], + "n_avg_mb": 0, + "n_copy_mb_s": 0, + "n_core_utilization": 0, + "n_cpu_percent_c": 0, + "n_cpu_percent_python": 0, + "n_gpu_avg_memory_mb": 0, + "n_gpu_peak_memory_mb": 0, + "n_gpu_percent": 0, + "n_growth_mb": 0, + "n_malloc_mb": 0, + "n_mallocs": 0, + "n_peak_mb": 0, + "n_python_fraction": 0, + "n_sys_percent": 0, + "n_usage_fraction": 0, + "start_outermost_loop": 12, + "start_region_line": 11 + }, + { + "cpu_samples_list": [], + "end_outermost_loop": 13, + "end_region_line": 13, + "line": "\n", + "lineno": 13, + "memory_samples": [], + "n_avg_mb": 0, + "n_copy_mb_s": 0, + "n_core_utilization": 0, + "n_cpu_percent_c": 0, + "n_cpu_percent_python": 0, + "n_gpu_avg_memory_mb": 0, + "n_gpu_peak_memory_mb": 0, + "n_gpu_percent": 0, + "n_growth_mb": 0, + "n_malloc_mb": 0, + "n_mallocs": 0, + "n_peak_mb": 0, + "n_python_fraction": 0, + "n_sys_percent": 0, + "n_usage_fraction": 0, + "start_outermost_loop": 13, + "start_region_line": 13 + }, + { + "cpu_samples_list": [], + "end_outermost_loop": 15, + "end_region_line": 14, + "line": "if __name__ == \"__main__\":\n", + "lineno": 14, + "memory_samples": [], + "n_avg_mb": 0, + "n_copy_mb_s": 0, + "n_core_utilization": 0, + "n_cpu_percent_c": 0, + "n_cpu_percent_python": 0, + "n_gpu_avg_memory_mb": 0, + "n_gpu_peak_memory_mb": 0, + "n_gpu_percent": 0, + "n_growth_mb": 0, + "n_malloc_mb": 0, + "n_mallocs": 0, + "n_peak_mb": 0, + "n_python_fraction": 0, + "n_sys_percent": 0, + "n_usage_fraction": 0, + "start_outermost_loop": 14, + "start_region_line": 14 + }, + { + "cpu_samples_list": [], + "end_outermost_loop": 15, + "end_region_line": 15, + "line": " my_function_to_profile()\n", + "lineno": 15, + "memory_samples": [], + "n_avg_mb": 0, + "n_copy_mb_s": 0, + "n_core_utilization": 0, + "n_cpu_percent_c": 0, + "n_cpu_percent_python": 0, + "n_gpu_avg_memory_mb": 0, + "n_gpu_peak_memory_mb": 0, + "n_gpu_percent": 0, + "n_growth_mb": 0, + "n_malloc_mb": 0, + "n_mallocs": 0, + "n_peak_mb": 0, + "n_python_fraction": 0, + "n_sys_percent": 0, + "n_usage_fraction": 0, + "start_outermost_loop": 15, + "start_region_line": 15 + } + ], + "percent_cpu_time": 100.0 + } + }, + "gpu": false, + "gpu_device": "", + "growth_rate": 0.0, + "max_footprint_fname": null, + "max_footprint_lineno": null, + "max_footprint_mb": 0, + "max_footprint_python_fraction": 0, + "memory": true, + "program": "/home/mohammed/Work/codeflash/codeflash/dummy.py", + "samples": [], + "stacks": [], + "start_time_absolute": 1757383905.0791636, + "start_time_perf": 8818.409090346 +} diff --git a/out.lprof b/out.lprof new file mode 100644 index 000000000..8c7b40ce3 --- /dev/null +++ b/out.lprof @@ -0,0 +1,26 @@ + /home/mohammed/Work/codeflash/codeflash/dummy.py: % of time = 100.00% (127.391ms) out of 127.391ms. + ╷ ╷ ╷ ╷ ╷ ╷ ╷ ╷ + │Time │–––––– │–––––– │Memory │–––––– │––––––––––– │Copy │ + Line │Python │native │system │Python │peak │timeline/% │(MB/s) │/home/mohammed/Work/codeflash/codeflash/dummy.py +╺━━━━━━┿━━━━━━━┿━━━━━━━┿━━━━━━━┿━━━━━━━━┿━━━━━━━┿━━━━━━━━━━━━━━━┿━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╸ + 1 │ │ │ │ │ │ │ │ + 2 │ │ │ │ │ │ │ │ + 3 │ │ │ │ │ │ │ │@profile + 4 │ │ │ │ │ │ │ │def my_function_to_profile(): + 5 │ │ │ │ │ │ │ │ # Your function's code here + 6 │ │ │ │ │ │ │ │ result = 0 + 7 │ │ │ │ │ │ │ │ for i in range(1000000): + 8 │ 85% │ 14% │ │ │ │ │ │ result += i * 2 + 9 │ │ │ │ │ │ │ │ return result + 10 │ │ │ │ │ │ │ │ + 11 │ │ │ │ │ │ │ │def another_function_we_dont_care_about(): + 12 │ │ │ │ │ │ │ │ pass + 13 │ │ │ │ │ │ │ │ + 14 │ │ │ │ │ │ │ │if __name__ == "__main__": + 15 │ │ │ │ │ │ │ │ my_function_to_profile() + 16 │ │ │ │ │ │ │ │ + │ │ │ │ │ │ │ │ +╶──────┼───────┼───────┼───────┼────────┼───────┼───────────────┼───────┼────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╴ + │ │ │ │ │ │ │ │function summary for /home/mohammed/Work/codeflash/codeflash/dummy.py + 3 │ 85% │ 14% │ │ │ │ │ │my_function_to_profile + ╵ ╵ ╵ ╵ ╵ ╵ ╵ ╵ diff --git a/tests/test_instrument_line_profiler.py b/tests/test_instrument_line_profiler.py index 1161bd7cd..7c3083f40 100644 --- a/tests/test_instrument_line_profiler.py +++ b/tests/test_instrument_line_profiler.py @@ -36,10 +36,9 @@ def test_add_decorator_imports_helper_in_class(): original_helper_code[helper_function_path] = helper_code line_profiler_output_file = add_decorator_imports( func_optimizer.function_to_optimize, code_context) - expected_code_main = f"""from line_profiler import profile as codeflash_line_profile -codeflash_line_profile.enable(output_prefix='{line_profiler_output_file}') - -from code_to_optimize.bubble_sort_in_class import BubbleSortClass + expected_code_main = f"""from code_to_optimize.bubble_sort_in_class import BubbleSortClass +from codeflash.code_utils.line_profile_utils import LineProfilerDecorator +codeflash_line_profile = LineProfilerDecorator('{line_profiler_output_file}') @codeflash_line_profile @@ -47,7 +46,8 @@ def sort_classmethod(x): y = BubbleSortClass() return y.sorter(x) """ - expected_code_helper = """from line_profiler import profile as codeflash_line_profile + expected_code_helper = """from codeflash.code_utils.line_profile_utils import LineProfilerDecorator +codeflash_line_profile = LineProfilerDecorator('{line_profiler_output_file}') def hi(): From ce899058d9d4d03cc70446a844817ba85aecc92c Mon Sep 17 00:00:00 2001 From: ali Date: Tue, 9 Dec 2025 14:04:54 +0200 Subject: [PATCH 03/10] reset --- code_to_optimize/find_common_tags.py | 3 - codeflash/code_utils/line_profile_utils.py | 88 ++-- codeflash/dummy.py | 15 - codeflash/dummy2.py | 78 ---- .../parse_line_profile_test_output.py | 3 +- out.json | 439 ------------------ out.lprof | 26 -- tests/test_instrument_line_profiler.py | 3 +- 8 files changed, 34 insertions(+), 621 deletions(-) delete mode 100644 codeflash/dummy.py delete mode 100644 codeflash/dummy2.py delete mode 100644 out.json delete mode 100644 out.lprof diff --git a/code_to_optimize/find_common_tags.py b/code_to_optimize/find_common_tags.py index 803ada4f0..016905bfc 100644 --- a/code_to_optimize/find_common_tags.py +++ b/code_to_optimize/find_common_tags.py @@ -1,9 +1,6 @@ from __future__ import annotations -from codeflash.code_utils.line_profile_utils import LineProfilerDecorator - -@codeflash_line_profile def find_common_tags(articles: list[dict[str, list[str]]]) -> set[str]: if not articles: return set() diff --git a/codeflash/code_utils/line_profile_utils.py b/codeflash/code_utils/line_profile_utils.py index 3c41e176d..27571dd0b 100644 --- a/codeflash/code_utils/line_profile_utils.py +++ b/codeflash/code_utils/line_profile_utils.py @@ -2,13 +2,11 @@ from __future__ import annotations -import functools from collections import defaultdict from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Union +from typing import TYPE_CHECKING, Union import libcst as cst -from line_profiler import LineProfiler from codeflash.code_utils.code_utils import get_run_tmp_file from codeflash.code_utils.formatter import sort_imports @@ -47,7 +45,6 @@ def leave_ClassDef(self, original_node: cst.ClassDef, updated_node: cst.ClassDef self.context_stack.pop() return updated_node - # f6fa22dd-d711-425b-bed4-d2844ad6af36 def visit_FunctionDef(self, node: cst.FunctionDef) -> None: # Track when we enter a function self.context_stack.append(node.name.value) @@ -81,17 +78,6 @@ def _is_target_decorator(self, decorator_node: Union[cst.Name, cst.Attribute, cs return False -# Helper to get full module path as string -def get_module_path(node: Union[cst.Name, cst.Attribute]) -> str: - parts = [] - while isinstance(node, cst.Attribute): - parts.insert(0, node.attr.value) - node = node.value - if isinstance(node, cst.Name): - parts.insert(0, node.value) - return ".".join(parts) - - class ProfileEnableTransformer(cst.CSTTransformer): def __init__(self, filename: str) -> None: # Flag to track if we found the import statement @@ -101,13 +87,21 @@ def __init__(self, filename: str) -> None: self.filename = filename def leave_ImportFrom(self, original_node: cst.ImportFrom, updated_node: cst.ImportFrom) -> cst.ImportFrom: - module_path = get_module_path(original_node.module) - - if module_path == "codeflash.code_utils.line_profile_utils" and any( - name.name.value == "LineProfilerDecorator" for name in original_node.names + # Check if this is the line profiler import statement + if ( + isinstance(original_node.module, cst.Name) + and original_node.module.value == "line_profiler" + and any( + name.name.value == "profile" and (not name.asname or name.asname.name.value == "codeflash_line_profile") + for name in original_node.names + ) ): - print("Found import statement") self.found_import = True + # Get the indentation from the original node + if hasattr(original_node, "leading_lines"): + leading_whitespace = original_node.leading_lines[-1].whitespace if original_node.leading_lines else "" + self.import_indentation = leading_whitespace + return updated_node def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> cst.Module: # noqa: ARG002 @@ -122,19 +116,26 @@ def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> c for i, stmt in enumerate(new_body): if isinstance(stmt, cst.SimpleStatementLine): for small_stmt in stmt.body: - if isinstance(small_stmt, cst.ImportFrom): - module_path = get_module_path(small_stmt.module) - if module_path == "codeflash.code_utils.line_profile_utils" and any( - name.name.value == "LineProfilerDecorator" for name in small_stmt.names - ): - import_index = i - break + if isinstance(small_stmt, cst.ImportFrom) and ( + isinstance(small_stmt.module, cst.Name) + and small_stmt.module.value == "line_profiler" + and any( + name.name.value == "profile" + and (not name.asname or name.asname.name.value == "codeflash_line_profile") + for name in small_stmt.names + ) + ): + import_index = i + break if import_index is not None: break if import_index is not None: - assignment = cst.parse_statement(f"codeflash_line_profile = LineProfilerDecorator('{self.filename}')") - new_body.insert(import_index + 1, assignment) + # Create the new enable statement to insert after the import + enable_statement = cst.parse_statement(f"codeflash_line_profile.enable(output_prefix='{self.filename}')") + + # Insert the new statement after the import statement + new_body.insert(import_index + 1, enable_statement) # Create a new module with the updated body return updated_node.with_changes(body=new_body) @@ -195,7 +196,7 @@ def add_decorator_imports(function_to_optimize: FunctionToOptimize, code_context # self.function_to_optimize, file_path_to_helper_classes, self.test_cfg.tests_root # grouped iteration, file to fns to optimize, from line_profiler import profile as codeflash_line_profile file_paths = defaultdict(list) - line_profile_output_file = get_run_tmp_file(Path("baseline_lprof.txt")) + line_profile_output_file = get_run_tmp_file(Path("baseline_lprof")) file_paths[function_to_optimize.file_path].append(function_to_optimize.qualified_name) for elem in code_context.helper_functions: file_paths[elem.file_path].append(elem.qualified_name) @@ -209,7 +210,7 @@ def add_decorator_imports(function_to_optimize: FunctionToOptimize, code_context module_node = add_decorator_to_qualified_function(module_node, fn_name, "codeflash_line_profile") # add imports # Create a transformer to add the import - transformer = ImportAdder("from codeflash.code_utils.line_profile_utils import LineProfilerDecorator") + transformer = ImportAdder("from line_profiler import profile as codeflash_line_profile") # Apply the transformer to add the import module_node = module_node.visit(transformer) modified_code = sort_imports(code=module_node.code, float_to_top=True) @@ -221,28 +222,3 @@ def add_decorator_imports(function_to_optimize: FunctionToOptimize, code_context modified_code = add_profile_enable(file_contents, line_profile_output_file.as_posix()) function_to_optimize.file_path.write_text(modified_code, "utf-8") return line_profile_output_file - - -class LineProfilerDecorator: - """Decorator class that profiles multiple functions and saves stats automatically.""" - - def __init__(self, output_file: str | Path) -> None: - self.output_file: Path = Path(output_file) - self.profiler: LineProfiler = LineProfiler() - # Ensure parent folder exists - self.output_file.parent.mkdir(parents=True, exist_ok=True) - - def __call__(self, func: Callable[..., Any]) -> Callable[..., Any]: - """Decorate a function to profile it and save stats after execution.""" - self.profiler.add_function(func) - - @functools.wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> Any: # noqa: ANN401 - try: - return self.profiler(func)(*args, **kwargs) - finally: - # Save stats after each call - with self.output_file.open("w") as f: - self.profiler.print_stats(f) - - return wrapper diff --git a/codeflash/dummy.py b/codeflash/dummy.py deleted file mode 100644 index b9468e19c..000000000 --- a/codeflash/dummy.py +++ /dev/null @@ -1,15 +0,0 @@ -# @profile -# def my_function_to_profile(): -# # Your function's code here -# result = 0 -# for i in range(1000000): -# result += i * 2 -# return result -# -# -# def another_function_we_dont_care_about(): -# pass -# -# -# if __name__ == "__main__": -# my_function_to_profile() diff --git a/codeflash/dummy2.py b/codeflash/dummy2.py deleted file mode 100644 index 9f6d8163c..000000000 --- a/codeflash/dummy2.py +++ /dev/null @@ -1,78 +0,0 @@ -# import random -# import time - -# from line_profiler import LineProfiler - -# codeflash_line_profile = LineProfiler() - - -# @codeflash_line_profile -# def complex_function(n: int) -> float: -# """A function with nested loops, math operations, and simulated I/O.""" - -# def helper_compute(x: float) -> float: -# """Nested helper doing some CPU-heavy math.""" -# total = 0.0 -# for i in range(1, 5000): -# total += math.sin(x * i) ** 2 + math.cos(x / (i + 1)) ** 3 -# return total - -# def helper_io_simulation(): -# """Simulate I/O-bound operations with tiny sleeps.""" -# for _ in range(5): -# time.sleep(0.001 * random.random()) # sleep for 0-1ms - -# result = 0.0 -# for i in range(1, n + 1): -# val = helper_compute(i) -# helper_io_simulation() -# result += val / (i + 1) - -# # some post-processing -# result = math.sqrt(result) + math.log1p(result) -# return result - -# complex_function(10) -# with open("foo.lprof", "w") as f: -# codeflash_line_profile.print_stats(f) - -# import functools -# from pathlib import Path - -# from line_profiler import LineProfiler - - -# class LineProfilerDecorator: -# """Decorator class that profiles multiple functions and saves stats automatically.""" -# def __init__(self, output_file: str | Path): -# self.output_file = Path(output_file) -# self.profiler = LineProfiler() -# # Ensure parent folder exists -# self.output_file.parent.mkdir(parents=True, exist_ok=True) - -# def __call__(self, func): -# """Decorate a function to profile it and save stats after execution.""" -# self.profiler.add_function(func) - -# @functools.wraps(func) -# def wrapper(*args, **kwargs): -# try: -# return self.profiler(func)(*args, **kwargs) -# finally: -# # Save stats after each call -# with self.output_file.open("w") as f: -# self.profiler.print_stats(f) - -# return wrapper - -# codeflash_line_profile = LineProfilerDecorator("baseline_lprof") - -# @codeflash_line_profile -# def foo(): -# for i in range(100): -# print("hello world") - -# if __name__ == "__main__": -# foo() - - diff --git a/codeflash/verification/parse_line_profile_test_output.py b/codeflash/verification/parse_line_profile_test_output.py index 177955c81..1877c0654 100644 --- a/codeflash/verification/parse_line_profile_test_output.py +++ b/codeflash/verification/parse_line_profile_test_output.py @@ -9,7 +9,6 @@ import dill as pickle -from codeflash.cli_cmds.console import logger from codeflash.code_utils.tabulate import tabulate if TYPE_CHECKING: @@ -79,9 +78,9 @@ def show_text(stats: dict) -> str: def parse_line_profile_results(line_profiler_output_file: Optional[Path]) -> dict: + line_profiler_output_file = line_profiler_output_file.with_suffix(".lprof") stats_dict = {} if not line_profiler_output_file.exists(): - logger.warning(f"Line profiler output file {line_profiler_output_file} does not exist.") return {"timings": {}, "unit": 0, "str_out": ""}, None with line_profiler_output_file.open("rb") as f: stats = pickle.load(f) diff --git a/out.json b/out.json deleted file mode 100644 index b8749e2a3..000000000 --- a/out.json +++ /dev/null @@ -1,439 +0,0 @@ -{ - "alloc_samples": 0, - "args": [ - "--reduce-profile", - "python", - "codeflash/dummy.py" - ], - "elapsed_time_sec": 0.1318800449371338, - "entrypoint_dir": "/home/mohammed/Work/codeflash/codeflash", - "filename": "/home/mohammed/Work/codeflash/codeflash", - "files": { - "/home/mohammed/Work/codeflash/codeflash/dummy.py": { - "functions": [ - { - "cpu_samples_list": [], - "line": "my_function_to_profile", - "lineno": 3, - "memory_samples": [], - "n_avg_mb": 0.0, - "n_copy_mb_s": 0.0, - "n_core_utilization": 0.12357065668313703, - "n_cpu_percent_c": 11.008220197077355, - "n_cpu_percent_python": 87.84830514943228, - "n_gpu_avg_memory_mb": 0.0, - "n_gpu_peak_memory_mb": 0.0, - "n_gpu_percent": 0.0, - "n_growth_mb": 0.0, - "n_malloc_mb": 0.0, - "n_mallocs": 0, - "n_peak_mb": 0.0, - "n_python_fraction": 0, - "n_sys_percent": 1.1434746534903728, - "n_usage_fraction": 0 - } - ], - "imports": [], - "leaks": {}, - "lines": [ - { - "cpu_samples_list": [], - "end_outermost_loop": 1, - "end_region_line": 1, - "line": "\n", - "lineno": 1, - "memory_samples": [], - "n_avg_mb": 0, - "n_copy_mb_s": 0, - "n_core_utilization": 0, - "n_cpu_percent_c": 0, - "n_cpu_percent_python": 0, - "n_gpu_avg_memory_mb": 0, - "n_gpu_peak_memory_mb": 0, - "n_gpu_percent": 0, - "n_growth_mb": 0, - "n_malloc_mb": 0, - "n_mallocs": 0, - "n_peak_mb": 0, - "n_python_fraction": 0, - "n_sys_percent": 0, - "n_usage_fraction": 0, - "start_outermost_loop": 1, - "start_region_line": 1 - }, - { - "cpu_samples_list": [], - "end_outermost_loop": 2, - "end_region_line": 2, - "line": "\n", - "lineno": 2, - "memory_samples": [], - "n_avg_mb": 0, - "n_copy_mb_s": 0, - "n_core_utilization": 0, - "n_cpu_percent_c": 0, - "n_cpu_percent_python": 0, - "n_gpu_avg_memory_mb": 0, - "n_gpu_peak_memory_mb": 0, - "n_gpu_percent": 0, - "n_growth_mb": 0, - "n_malloc_mb": 0, - "n_mallocs": 0, - "n_peak_mb": 0, - "n_python_fraction": 0, - "n_sys_percent": 0, - "n_usage_fraction": 0, - "start_outermost_loop": 2, - "start_region_line": 2 - }, - { - "cpu_samples_list": [], - "end_outermost_loop": 3, - "end_region_line": 3, - "line": "@profile\n", - "lineno": 3, - "memory_samples": [], - "n_avg_mb": 0.0, - "n_copy_mb_s": 0.0, - "n_core_utilization": 0.0, - "n_cpu_percent_c": 0.0, - "n_cpu_percent_python": 0.0, - "n_gpu_avg_memory_mb": 0.0, - "n_gpu_peak_memory_mb": 0.0, - "n_gpu_percent": 0, - "n_growth_mb": 0.0, - "n_malloc_mb": 0.0, - "n_mallocs": 0, - "n_peak_mb": 0.0, - "n_python_fraction": 0, - "n_sys_percent": 0.0, - "n_usage_fraction": 0, - "start_outermost_loop": 3, - "start_region_line": 3 - }, - { - "cpu_samples_list": [], - "end_outermost_loop": 9, - "end_region_line": 9, - "line": "def my_function_to_profile():\n", - "lineno": 4, - "memory_samples": [], - "n_avg_mb": 0.0, - "n_copy_mb_s": 0.0, - "n_core_utilization": 0.0, - "n_cpu_percent_c": 0.0, - "n_cpu_percent_python": 0.0, - "n_gpu_avg_memory_mb": 0.0, - "n_gpu_peak_memory_mb": 0.0, - "n_gpu_percent": 0, - "n_growth_mb": 0.0, - "n_malloc_mb": 0.0, - "n_mallocs": 0, - "n_peak_mb": 0.0, - "n_python_fraction": 0, - "n_sys_percent": 0.0, - "n_usage_fraction": 0, - "start_outermost_loop": 4, - "start_region_line": 4 - }, - { - "cpu_samples_list": [], - "end_outermost_loop": 9, - "end_region_line": 9, - "line": " # Your function's code here\n", - "lineno": 5, - "memory_samples": [], - "n_avg_mb": 0.0, - "n_copy_mb_s": 0.0, - "n_core_utilization": 0.0, - "n_cpu_percent_c": 0.0, - "n_cpu_percent_python": 0.0, - "n_gpu_avg_memory_mb": 0.0, - "n_gpu_peak_memory_mb": 0.0, - "n_gpu_percent": 0, - "n_growth_mb": 0.0, - "n_malloc_mb": 0.0, - "n_mallocs": 0, - "n_peak_mb": 0.0, - "n_python_fraction": 0, - "n_sys_percent": 0.0, - "n_usage_fraction": 0, - "start_outermost_loop": 4, - "start_region_line": 4 - }, - { - "cpu_samples_list": [], - "end_outermost_loop": 6, - "end_region_line": 9, - "line": " result = 0\n", - "lineno": 6, - "memory_samples": [], - "n_avg_mb": 0.0, - "n_copy_mb_s": 0.0, - "n_core_utilization": 0.0, - "n_cpu_percent_c": 0.0, - "n_cpu_percent_python": 0.0, - "n_gpu_avg_memory_mb": 0.0, - "n_gpu_peak_memory_mb": 0.0, - "n_gpu_percent": 0, - "n_growth_mb": 0.0, - "n_malloc_mb": 0.0, - "n_mallocs": 0, - "n_peak_mb": 0.0, - "n_python_fraction": 0, - "n_sys_percent": 0.0, - "n_usage_fraction": 0, - "start_outermost_loop": 6, - "start_region_line": 4 - }, - { - "cpu_samples_list": [], - "end_outermost_loop": 8, - "end_region_line": 8, - "line": " for i in range(1000000):\n", - "lineno": 7, - "memory_samples": [], - "n_avg_mb": 0.0, - "n_copy_mb_s": 0.0, - "n_core_utilization": 0.0, - "n_cpu_percent_c": 0.0, - "n_cpu_percent_python": 0.0, - "n_gpu_avg_memory_mb": 0.0, - "n_gpu_peak_memory_mb": 0.0, - "n_gpu_percent": 0, - "n_growth_mb": 0.0, - "n_malloc_mb": 0.0, - "n_mallocs": 0, - "n_peak_mb": 0.0, - "n_python_fraction": 0, - "n_sys_percent": 0.0, - "n_usage_fraction": 0, - "start_outermost_loop": 7, - "start_region_line": 7 - }, - { - "cpu_samples_list": [ - 8818.4491195, - 8818.468866728, - 8818.46997304, - 8818.480837068, - 8818.486153326, - 8818.493161905, - 8818.51569137 - ], - "end_outermost_loop": 8, - "end_region_line": 8, - "line": " result += i * 2\n", - "lineno": 8, - "memory_samples": [], - "n_avg_mb": 0.0, - "n_copy_mb_s": 0.0, - "n_core_utilization": 0.12357065668313703, - "n_cpu_percent_c": 11.008220197077355, - "n_cpu_percent_python": 87.84830514943228, - "n_gpu_avg_memory_mb": 0.0, - "n_gpu_peak_memory_mb": 0.0, - "n_gpu_percent": 0.0, - "n_growth_mb": 0.0, - "n_malloc_mb": 0.0, - "n_mallocs": 0, - "n_peak_mb": 0.0, - "n_python_fraction": 0, - "n_sys_percent": 1.1434746534903728, - "n_usage_fraction": 0, - "start_outermost_loop": 7, - "start_region_line": 7 - }, - { - "cpu_samples_list": [], - "end_outermost_loop": 9, - "end_region_line": 9, - "line": " return result\n", - "lineno": 9, - "memory_samples": [], - "n_avg_mb": 0.0, - "n_copy_mb_s": 0.0, - "n_core_utilization": 0.0, - "n_cpu_percent_c": 0.0, - "n_cpu_percent_python": 0.0, - "n_gpu_avg_memory_mb": 0.0, - "n_gpu_peak_memory_mb": 0.0, - "n_gpu_percent": 0, - "n_growth_mb": 0.0, - "n_malloc_mb": 0.0, - "n_mallocs": 0, - "n_peak_mb": 0.0, - "n_python_fraction": 0, - "n_sys_percent": 0.0, - "n_usage_fraction": 0, - "start_outermost_loop": 9, - "start_region_line": 4 - }, - { - "cpu_samples_list": [], - "end_outermost_loop": 10, - "end_region_line": 10, - "line": "\n", - "lineno": 10, - "memory_samples": [], - "n_avg_mb": 0, - "n_copy_mb_s": 0, - "n_core_utilization": 0, - "n_cpu_percent_c": 0, - "n_cpu_percent_python": 0, - "n_gpu_avg_memory_mb": 0, - "n_gpu_peak_memory_mb": 0, - "n_gpu_percent": 0, - "n_growth_mb": 0, - "n_malloc_mb": 0, - "n_mallocs": 0, - "n_peak_mb": 0, - "n_python_fraction": 0, - "n_sys_percent": 0, - "n_usage_fraction": 0, - "start_outermost_loop": 10, - "start_region_line": 10 - }, - { - "cpu_samples_list": [], - "end_outermost_loop": 12, - "end_region_line": 12, - "line": "def another_function_we_dont_care_about():\n", - "lineno": 11, - "memory_samples": [], - "n_avg_mb": 0, - "n_copy_mb_s": 0, - "n_core_utilization": 0, - "n_cpu_percent_c": 0, - "n_cpu_percent_python": 0, - "n_gpu_avg_memory_mb": 0, - "n_gpu_peak_memory_mb": 0, - "n_gpu_percent": 0, - "n_growth_mb": 0, - "n_malloc_mb": 0, - "n_mallocs": 0, - "n_peak_mb": 0, - "n_python_fraction": 0, - "n_sys_percent": 0, - "n_usage_fraction": 0, - "start_outermost_loop": 11, - "start_region_line": 11 - }, - { - "cpu_samples_list": [], - "end_outermost_loop": 12, - "end_region_line": 12, - "line": " pass\n", - "lineno": 12, - "memory_samples": [], - "n_avg_mb": 0, - "n_copy_mb_s": 0, - "n_core_utilization": 0, - "n_cpu_percent_c": 0, - "n_cpu_percent_python": 0, - "n_gpu_avg_memory_mb": 0, - "n_gpu_peak_memory_mb": 0, - "n_gpu_percent": 0, - "n_growth_mb": 0, - "n_malloc_mb": 0, - "n_mallocs": 0, - "n_peak_mb": 0, - "n_python_fraction": 0, - "n_sys_percent": 0, - "n_usage_fraction": 0, - "start_outermost_loop": 12, - "start_region_line": 11 - }, - { - "cpu_samples_list": [], - "end_outermost_loop": 13, - "end_region_line": 13, - "line": "\n", - "lineno": 13, - "memory_samples": [], - "n_avg_mb": 0, - "n_copy_mb_s": 0, - "n_core_utilization": 0, - "n_cpu_percent_c": 0, - "n_cpu_percent_python": 0, - "n_gpu_avg_memory_mb": 0, - "n_gpu_peak_memory_mb": 0, - "n_gpu_percent": 0, - "n_growth_mb": 0, - "n_malloc_mb": 0, - "n_mallocs": 0, - "n_peak_mb": 0, - "n_python_fraction": 0, - "n_sys_percent": 0, - "n_usage_fraction": 0, - "start_outermost_loop": 13, - "start_region_line": 13 - }, - { - "cpu_samples_list": [], - "end_outermost_loop": 15, - "end_region_line": 14, - "line": "if __name__ == \"__main__\":\n", - "lineno": 14, - "memory_samples": [], - "n_avg_mb": 0, - "n_copy_mb_s": 0, - "n_core_utilization": 0, - "n_cpu_percent_c": 0, - "n_cpu_percent_python": 0, - "n_gpu_avg_memory_mb": 0, - "n_gpu_peak_memory_mb": 0, - "n_gpu_percent": 0, - "n_growth_mb": 0, - "n_malloc_mb": 0, - "n_mallocs": 0, - "n_peak_mb": 0, - "n_python_fraction": 0, - "n_sys_percent": 0, - "n_usage_fraction": 0, - "start_outermost_loop": 14, - "start_region_line": 14 - }, - { - "cpu_samples_list": [], - "end_outermost_loop": 15, - "end_region_line": 15, - "line": " my_function_to_profile()\n", - "lineno": 15, - "memory_samples": [], - "n_avg_mb": 0, - "n_copy_mb_s": 0, - "n_core_utilization": 0, - "n_cpu_percent_c": 0, - "n_cpu_percent_python": 0, - "n_gpu_avg_memory_mb": 0, - "n_gpu_peak_memory_mb": 0, - "n_gpu_percent": 0, - "n_growth_mb": 0, - "n_malloc_mb": 0, - "n_mallocs": 0, - "n_peak_mb": 0, - "n_python_fraction": 0, - "n_sys_percent": 0, - "n_usage_fraction": 0, - "start_outermost_loop": 15, - "start_region_line": 15 - } - ], - "percent_cpu_time": 100.0 - } - }, - "gpu": false, - "gpu_device": "", - "growth_rate": 0.0, - "max_footprint_fname": null, - "max_footprint_lineno": null, - "max_footprint_mb": 0, - "max_footprint_python_fraction": 0, - "memory": true, - "program": "/home/mohammed/Work/codeflash/codeflash/dummy.py", - "samples": [], - "stacks": [], - "start_time_absolute": 1757383905.0791636, - "start_time_perf": 8818.409090346 -} diff --git a/out.lprof b/out.lprof deleted file mode 100644 index 8c7b40ce3..000000000 --- a/out.lprof +++ /dev/null @@ -1,26 +0,0 @@ - /home/mohammed/Work/codeflash/codeflash/dummy.py: % of time = 100.00% (127.391ms) out of 127.391ms. - ╷ ╷ ╷ ╷ ╷ ╷ ╷ ╷ - │Time │–––––– │–––––– │Memory │–––––– │––––––––––– │Copy │ - Line │Python │native │system │Python │peak │timeline/% │(MB/s) │/home/mohammed/Work/codeflash/codeflash/dummy.py -╺━━━━━━┿━━━━━━━┿━━━━━━━┿━━━━━━━┿━━━━━━━━┿━━━━━━━┿━━━━━━━━━━━━━━━┿━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╸ - 1 │ │ │ │ │ │ │ │ - 2 │ │ │ │ │ │ │ │ - 3 │ │ │ │ │ │ │ │@profile - 4 │ │ │ │ │ │ │ │def my_function_to_profile(): - 5 │ │ │ │ │ │ │ │ # Your function's code here - 6 │ │ │ │ │ │ │ │ result = 0 - 7 │ │ │ │ │ │ │ │ for i in range(1000000): - 8 │ 85% │ 14% │ │ │ │ │ │ result += i * 2 - 9 │ │ │ │ │ │ │ │ return result - 10 │ │ │ │ │ │ │ │ - 11 │ │ │ │ │ │ │ │def another_function_we_dont_care_about(): - 12 │ │ │ │ │ │ │ │ pass - 13 │ │ │ │ │ │ │ │ - 14 │ │ │ │ │ │ │ │if __name__ == "__main__": - 15 │ │ │ │ │ │ │ │ my_function_to_profile() - 16 │ │ │ │ │ │ │ │ - │ │ │ │ │ │ │ │ -╶──────┼───────┼───────┼───────┼────────┼───────┼───────────────┼───────┼────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╴ - │ │ │ │ │ │ │ │function summary for /home/mohammed/Work/codeflash/codeflash/dummy.py - 3 │ 85% │ 14% │ │ │ │ │ │my_function_to_profile - ╵ ╵ ╵ ╵ ╵ ╵ ╵ ╵ diff --git a/tests/test_instrument_line_profiler.py b/tests/test_instrument_line_profiler.py index d4493c923..71d1005c0 100644 --- a/tests/test_instrument_line_profiler.py +++ b/tests/test_instrument_line_profiler.py @@ -47,8 +47,7 @@ def sort_classmethod(x): y = BubbleSortClass() return y.sorter(x) """ - expected_code_helper = """from codeflash.code_utils.line_profile_utils import LineProfilerDecorator -codeflash_line_profile = LineProfilerDecorator('{line_profiler_output_file}') + expected_code_helper = """from line_profiler import profile as codeflash_line_profile def hi(): From 1f367bedc8bb4fe2dfb61a388ec4e73aea79202b Mon Sep 17 00:00:00 2001 From: ali Date: Tue, 9 Dec 2025 14:48:52 +0200 Subject: [PATCH 04/10] dynamic tolerance --- codeflash/verification/pytest_plugin.py | 29 ++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/codeflash/verification/pytest_plugin.py b/codeflash/verification/pytest_plugin.py index 513e5f8b7..c8a38719e 100644 --- a/codeflash/verification/pytest_plugin.py +++ b/codeflash/verification/pytest_plugin.py @@ -8,9 +8,11 @@ import os import platform import re +import statistics import sys import time as _time_module import warnings +from collections import deque from pathlib import Path from typing import TYPE_CHECKING, Any, Callable from unittest import TestCase @@ -271,9 +273,24 @@ def __init__(self, config: Config) -> None: logging.basicConfig(level=level) self.logger = logging.getLogger(self.name) + def dynamic_tolerance(self, avg: float) -> float: + # (< 0.1 ms) + if avg < 0.0001: + return 0.7 # 70% + + # (< 0.5 ms) + if avg < 0.0005: + return 0.4 # 40% + + # (< 1 ms) + if avg < 0.001: + return 0.2 # 20% + + return CONSISTENT_DURATION_TOLERANCE + @hookspec(firstresult=True) def pytest_runtestloop(self, session: Session) -> bool: - durations: list[float] = [] + durations = deque(maxlen=CONSISTENT_LOOP_COUNT) """Reimplement the test loop but loop for the user defined amount of time.""" if session.testsfailed and not session.config.option.continue_on_collection_errors: @@ -314,10 +331,12 @@ def pytest_runtestloop(self, session: Session) -> bool: durations.append(loop_duration) # Consistency check - if len(durations) >= CONSISTENT_LOOP_COUNT: - recent = durations[-CONSISTENT_LOOP_COUNT:] - avg = sum(recent) / CONSISTENT_LOOP_COUNT - consistent = all(abs(d - avg) / avg <= CONSISTENT_DURATION_TOLERANCE for d in recent) + if len(durations) == CONSISTENT_LOOP_COUNT: + avg = statistics.median(durations) + if avg == 0: + consistent = all(d == 0 for d in durations) + else: + consistent = all(abs(d - avg) / avg <= self.dynamic_tolerance(avg) for d in durations) if consistent: break From 5c4a6d952226e8698aa893534bbf897a07079436 Mon Sep 17 00:00:00 2001 From: ali Date: Fri, 12 Dec 2025 04:44:31 +0200 Subject: [PATCH 05/10] get the duration from the pytest overriden methods --- codeflash/code_utils/config_consts.py | 3 +- codeflash/code_utils/env_utils.py | 1 - codeflash/verification/pytest_plugin.py | 48 ++++++++++++++----------- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/codeflash/code_utils/config_consts.py b/codeflash/code_utils/config_consts.py index 22b277427..638655632 100644 --- a/codeflash/code_utils/config_consts.py +++ b/codeflash/code_utils/config_consts.py @@ -8,8 +8,7 @@ MAX_CUMULATIVE_TEST_RUNTIME_NANOSECONDS = 100e6 # 100ms N_TESTS_TO_GENERATE = 2 TOTAL_LOOPING_TIME = 10.0 # 10 second candidate benchmarking budget -CONSISTENT_LOOP_COUNT = 5 -CONSISTENT_DURATION_TOLERANCE = 0.05 +CONSISTENT_LOOP_COUNT = 3 COVERAGE_THRESHOLD = 60.0 MIN_TESTCASE_PASSED_THRESHOLD = 6 REPEAT_OPTIMIZATION_PROBABILITY = 0.1 diff --git a/codeflash/code_utils/env_utils.py b/codeflash/code_utils/env_utils.py index 4987e6d8d..32cbdfa2b 100644 --- a/codeflash/code_utils/env_utils.py +++ b/codeflash/code_utils/env_utils.py @@ -19,7 +19,6 @@ def check_formatter_installed(formatter_cmds: list[str], exit_on_failure: bool = True) -> bool: # noqa if not formatter_cmds or formatter_cmds[0] == "disabled": return True - first_cmd = formatter_cmds[0] cmd_tokens = shlex.split(first_cmd) if isinstance(first_cmd, str) else [first_cmd] diff --git a/codeflash/verification/pytest_plugin.py b/codeflash/verification/pytest_plugin.py index c8a38719e..1b0bef261 100644 --- a/codeflash/verification/pytest_plugin.py +++ b/codeflash/verification/pytest_plugin.py @@ -2,8 +2,6 @@ import contextlib import inspect - -# System Imports import logging import os import platform @@ -13,6 +11,8 @@ import time as _time_module import warnings from collections import deque + +# System Imports from pathlib import Path from typing import TYPE_CHECKING, Any, Callable from unittest import TestCase @@ -21,7 +21,7 @@ import pytest from pluggy import HookspecMarker -from codeflash.code_utils.config_consts import CONSISTENT_DURATION_TOLERANCE, CONSISTENT_LOOP_COUNT +from codeflash.code_utils.config_consts import CONSISTENT_LOOP_COUNT if TYPE_CHECKING: from _pytest.config import Config, Parser @@ -272,21 +272,25 @@ def __init__(self, config: Config) -> None: level = logging.DEBUG if config.option.verbose > 1 else logging.INFO logging.basicConfig(level=level) self.logger = logging.getLogger(self.name) + self.current_loop_durations_in_seconds: list[float] = [] def dynamic_tolerance(self, avg: float) -> float: - # (< 0.1 ms) - if avg < 0.0001: - return 0.7 # 70% - - # (< 0.5 ms) - if avg < 0.0005: - return 0.4 # 40% - - # (< 1 ms) - if avg < 0.001: - return 0.2 # 20% - - return CONSISTENT_DURATION_TOLERANCE + if avg < 0.0001: # < 100 µs + return 0.7 + if avg < 0.0005: # < 500 µs + return 0.5 + if avg < 0.001: # < 1 ms + return 0.4 + if avg < 0.01: # < 10 ms + return 0.2 + if avg < 0.1: # < 100 ms + return 0.1 + return 0.03 # > 0.1 s + + @pytest.hookimpl + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + if report.when == "call" and report.outcome == "passed": + self.current_loop_durations_in_seconds.append(report.duration) @hookspec(firstresult=True) def pytest_runtestloop(self, session: Session) -> bool: @@ -307,7 +311,7 @@ def pytest_runtestloop(self, session: Session) -> bool: while total_time >= SHORTEST_AMOUNT_OF_TIME: count += 1 - loop_start = _ORIGINAL_TIME_TIME() + self.current_loop_durations_in_seconds.clear() for index, item in enumerate(session.items): item: pytest.Item = item # noqa: PLW0127, PLW2901 @@ -326,9 +330,12 @@ def pytest_runtestloop(self, session: Session) -> bool: if session.shouldstop: raise session.Interrupted(session.shouldstop) - loop_end = _ORIGINAL_TIME_TIME() - loop_duration = loop_end - loop_start - durations.append(loop_duration) + total_duration_in_seconds = sum(self.current_loop_durations_in_seconds) + + if total_duration_in_seconds > 0: + durations.append(total_duration_in_seconds) + else: + durations.clear() # Consistency check if len(durations) == CONSISTENT_LOOP_COUNT: @@ -338,6 +345,7 @@ def pytest_runtestloop(self, session: Session) -> bool: else: consistent = all(abs(d - avg) / avg <= self.dynamic_tolerance(avg) for d in durations) if consistent: + Path(f"/home/mohammed/Documents/test-results/break-{session.name}.txt").write_text(str(count)) break if self._timed_out(session, start_time, count): From ecd21d5d84effed52d32c26846ff243588f48ac2 Mon Sep 17 00:00:00 2001 From: ali Date: Fri, 12 Dec 2025 04:46:18 +0200 Subject: [PATCH 06/10] remove debug log --- codeflash/verification/pytest_plugin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/codeflash/verification/pytest_plugin.py b/codeflash/verification/pytest_plugin.py index 1b0bef261..fd3d19644 100644 --- a/codeflash/verification/pytest_plugin.py +++ b/codeflash/verification/pytest_plugin.py @@ -345,7 +345,6 @@ def pytest_runtestloop(self, session: Session) -> bool: else: consistent = all(abs(d - avg) / avg <= self.dynamic_tolerance(avg) for d in durations) if consistent: - Path(f"/home/mohammed/Documents/test-results/break-{session.name}.txt").write_text(str(count)) break if self._timed_out(session, start_time, count): From 30c89ce48b5b31fb9e8eece382b64235411099db Mon Sep 17 00:00:00 2001 From: ali Date: Fri, 12 Dec 2025 06:46:54 +0200 Subject: [PATCH 07/10] respect the min loop count -just in case- --- codeflash/optimization/function_optimizer.py | 1 - codeflash/verification/pytest_plugin.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/codeflash/optimization/function_optimizer.py b/codeflash/optimization/function_optimizer.py index 3fda86489..6185be2ef 100644 --- a/codeflash/optimization/function_optimizer.py +++ b/codeflash/optimization/function_optimizer.py @@ -1804,7 +1804,6 @@ def establish_original_code_baseline( benchmarking_results, self.function_to_optimize.function_name ) logger.debug(f"Original async function throughput: {async_throughput} calls/second") - console.rule() if self.args.benchmark: replay_benchmarking_test_results = benchmarking_results.group_by_benchmarks( diff --git a/codeflash/verification/pytest_plugin.py b/codeflash/verification/pytest_plugin.py index fd3d19644..48ad6e5ae 100644 --- a/codeflash/verification/pytest_plugin.py +++ b/codeflash/verification/pytest_plugin.py @@ -338,7 +338,7 @@ def pytest_runtestloop(self, session: Session) -> bool: durations.clear() # Consistency check - if len(durations) == CONSISTENT_LOOP_COUNT: + if len(durations) == CONSISTENT_LOOP_COUNT and count >= session.config.option.codeflash_min_loops: avg = statistics.median(durations) if avg == 0: consistent = all(d == 0 for d in durations) From 89fc93994e147e3789cbc5ad9a9bf3c22ea09774 Mon Sep 17 00:00:00 2001 From: ali Date: Mon, 15 Dec 2025 20:17:09 +0200 Subject: [PATCH 08/10] more closer method --- codeflash/verification/pytest_plugin.py | 43 +++++++++++++++++++------ 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/codeflash/verification/pytest_plugin.py b/codeflash/verification/pytest_plugin.py index 48ad6e5ae..cfad951d7 100644 --- a/codeflash/verification/pytest_plugin.py +++ b/codeflash/verification/pytest_plugin.py @@ -81,6 +81,7 @@ class UnexpectedError(Exception): # Store references to original functions before any patching _ORIGINAL_TIME_TIME = _time_module.time _ORIGINAL_PERF_COUNTER = _time_module.perf_counter +_ORIGINAL_PERF_COUNTER_NS = _time_module.perf_counter_ns _ORIGINAL_TIME_SLEEP = _time_module.sleep @@ -272,7 +273,7 @@ def __init__(self, config: Config) -> None: level = logging.DEBUG if config.option.verbose > 1 else logging.INFO logging.basicConfig(level=level) self.logger = logging.getLogger(self.name) - self.current_loop_durations_in_seconds: list[float] = [] + self.current_loop_durations_in_nano: list[int] = [] def dynamic_tolerance(self, avg: float) -> float: if avg < 0.0001: # < 100 µs @@ -287,10 +288,31 @@ def dynamic_tolerance(self, avg: float) -> float: return 0.1 return 0.03 # > 0.1 s - @pytest.hookimpl - def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: - if report.when == "call" and report.outcome == "passed": - self.current_loop_durations_in_seconds.append(report.duration) + # @pytest.hookimpl + # def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + # if report.when == "call" and report.passed: + # self.current_loop_durations_in_nano.append(report.duration) + + @pytest.hookimpl(tryfirst=True) + def pytest_pyfunc_call(self, pyfuncitem: pytest.Function) -> bool: + testfunction = pyfuncitem.obj + funcargs = pyfuncitem.funcargs + testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames} # noqa: SLF001 + + start_ns = _ORIGINAL_PERF_COUNTER_NS() + result = testfunction(**testargs) + duration_ns = _ORIGINAL_PERF_COUNTER_NS() - start_ns + + self.current_loop_durations_in_nano.append(duration_ns) + + # original post-processing + if hasattr(result, "__await__") or hasattr(result, "__aiter__"): + msg = f"Async test not supported: {pyfuncitem.nodeid}" + raise RuntimeError(msg) + if result is not None: + warnings.warn(f"Test function {pyfuncitem.nodeid} returned {type(result)}, expected None.", stacklevel=2) + + return True @hookspec(firstresult=True) def pytest_runtestloop(self, session: Session) -> bool: @@ -308,10 +330,10 @@ def pytest_runtestloop(self, session: Session) -> bool: total_time: float = self._get_total_time(session) count: int = 0 - + runtimes = [] while total_time >= SHORTEST_AMOUNT_OF_TIME: count += 1 - self.current_loop_durations_in_seconds.clear() + self.current_loop_durations_in_nano.clear() for index, item in enumerate(session.items): item: pytest.Item = item # noqa: PLW0127, PLW2901 @@ -330,10 +352,11 @@ def pytest_runtestloop(self, session: Session) -> bool: if session.shouldstop: raise session.Interrupted(session.shouldstop) - total_duration_in_seconds = sum(self.current_loop_durations_in_seconds) + runtimes.extend(list(self.current_loop_durations_in_nano)) - if total_duration_in_seconds > 0: - durations.append(total_duration_in_seconds) + total_duration_in_nano = sum(self.current_loop_durations_in_nano) + if total_duration_in_nano > 0: + durations.append(total_duration_in_nano) else: durations.clear() From a67dad3e15824379a63760ccdc5412200b750084 Mon Sep 17 00:00:00 2001 From: ali Date: Tue, 16 Dec 2025 04:47:46 +0200 Subject: [PATCH 09/10] working version --- codeflash/verification/pytest_plugin.py | 43 +++++++++++-------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/codeflash/verification/pytest_plugin.py b/codeflash/verification/pytest_plugin.py index cfad951d7..5d5f8ec7f 100644 --- a/codeflash/verification/pytest_plugin.py +++ b/codeflash/verification/pytest_plugin.py @@ -288,31 +288,24 @@ def dynamic_tolerance(self, avg: float) -> float: return 0.1 return 0.03 # > 0.1 s - # @pytest.hookimpl - # def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: - # if report.when == "call" and report.passed: - # self.current_loop_durations_in_nano.append(report.duration) - - @pytest.hookimpl(tryfirst=True) - def pytest_pyfunc_call(self, pyfuncitem: pytest.Function) -> bool: - testfunction = pyfuncitem.obj - funcargs = pyfuncitem.funcargs - testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames} # noqa: SLF001 - - start_ns = _ORIGINAL_PERF_COUNTER_NS() - result = testfunction(**testargs) - duration_ns = _ORIGINAL_PERF_COUNTER_NS() - start_ns - - self.current_loop_durations_in_nano.append(duration_ns) - - # original post-processing - if hasattr(result, "__await__") or hasattr(result, "__aiter__"): - msg = f"Async test not supported: {pyfuncitem.nodeid}" - raise RuntimeError(msg) - if result is not None: - warnings.warn(f"Test function {pyfuncitem.nodeid} returned {type(result)}, expected None.", stacklevel=2) - - return True + @pytest.hookimpl + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + if report.when == "call": + stdout = report.capstdout + i = len(stdout) + + # Skip trailing newlines + while i and stdout[i - 1] == "\n": + i -= 1 + + if i: + j = stdout.rfind("\n", 0, i) + last_line = stdout[j + 1 : i] + + if last_line[:7] == "!######": + last_colon = last_line.rfind(":", 0, last_line.rfind("######")) + duration = last_line[last_colon + 1 : last_line.rfind("######")] + self.current_loop_durations_in_nano.append(int(duration)) @hookspec(firstresult=True) def pytest_runtestloop(self, session: Session) -> bool: From d52aae4b53d8a38ce15b5a30853ef667b258d70d Mon Sep 17 00:00:00 2001 From: ali Date: Tue, 16 Dec 2025 05:28:20 +0200 Subject: [PATCH 10/10] even better --- codeflash/verification/pytest_plugin.py | 46 ++++++++++++++++--------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/codeflash/verification/pytest_plugin.py b/codeflash/verification/pytest_plugin.py index 5d5f8ec7f..791a39ec4 100644 --- a/codeflash/verification/pytest_plugin.py +++ b/codeflash/verification/pytest_plugin.py @@ -14,7 +14,7 @@ # System Imports from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable +from typing import TYPE_CHECKING, Any, Callable, Optional from unittest import TestCase # PyTest Imports @@ -265,6 +265,29 @@ def pytest_configure(config: Config) -> None: _apply_deterministic_patches() +def get_runtime_from_stdout(stdout: str) -> Optional[int]: + marker_start = "!######" + marker_end = "######!" + + if not stdout: + return None + + end = stdout.rfind(marker_end) + if end == -1: + return None + + start = stdout.rfind(marker_start, 0, end) + if start == -1: + return None + + payload = stdout[start + len(marker_start) : end] + last_colon = payload.rfind(":") + if last_colon == -1: + return None + + return int(payload[last_colon + 1 :]) + + class PytestLoops: name: str = "pytest-loops" @@ -290,22 +313,8 @@ def dynamic_tolerance(self, avg: float) -> float: @pytest.hookimpl def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: - if report.when == "call": - stdout = report.capstdout - i = len(stdout) - - # Skip trailing newlines - while i and stdout[i - 1] == "\n": - i -= 1 - - if i: - j = stdout.rfind("\n", 0, i) - last_line = stdout[j + 1 : i] - - if last_line[:7] == "!######": - last_colon = last_line.rfind(":", 0, last_line.rfind("######")) - duration = last_line[last_colon + 1 : last_line.rfind("######")] - self.current_loop_durations_in_nano.append(int(duration)) + if report.when == "call" and (duration_ns := get_runtime_from_stdout(report.capstdout)): + self.current_loop_durations_in_nano.append(duration_ns) @hookspec(firstresult=True) def pytest_runtestloop(self, session: Session) -> bool: @@ -361,6 +370,9 @@ def pytest_runtestloop(self, session: Session) -> bool: else: consistent = all(abs(d - avg) / avg <= self.dynamic_tolerance(avg) for d in durations) if consistent: + Path(f"/home/mohammed/Documents/test-results/break-{int(_ORIGINAL_TIME_TIME())}.txt").write_text( + f"loops: {count}, runtime: {runtimes}" + ) break if self._timed_out(session, start_time, count):