diff --git a/docs/docs/cli-reference.md b/docs/docs/cli-reference.md index 50e5242..493d74c 100644 --- a/docs/docs/cli-reference.md +++ b/docs/docs/cli-reference.md @@ -12,8 +12,9 @@ Complete command reference for OracleTrace. ```bash oracletrace [--json OUTPUT.json] [--csv OUTPUT.csv] [--compare BASELINE.json] -oracletrace [--ignore REGEX [REGEX ...]] +oracletrace [--ignore REGEX [REGEX ...]] oracletrace [--top NUMBER] +oracletrace [--compare BASELINE.json] [--fail-on-regression] [--threshold PERCENT] ``` ## Required argument @@ -62,6 +63,24 @@ Comparison output includes: - Newly added functions - Removed functions +### `--fail-on-regression` + +Makes OracleTrace return a non-zero exit code when a regression exceeds the configured threshold. + +Example: + +```bash +oracletrace my_app.py --json current.json --compare baseline.json --fail-on-regression --threshold 25 +``` + +### `--threshold` + +Sets the percentage threshold used with `--fail-on-regression`. + +```bash +oracletrace my_app.py --compare baseline.json --fail-on-regression --threshold 25 +``` + ### `--ignore` Specify file paths and function names to ignore using regular expression syntax. @@ -86,6 +105,7 @@ OracleTrace returns a non-zero exit code when: - The target script does not exist - The compare JSON file does not exist +- `--fail-on-regression` is enabled and at least one function regresses above the threshold Non-zero exit codes can also happen when the target script fails at runtime or when the compare JSON file cannot be parsed. diff --git a/docs/docs/examples.md b/docs/docs/examples.md index 3bf35a7..053b16e 100644 --- a/docs/docs/examples.md +++ b/docs/docs/examples.md @@ -50,6 +50,15 @@ oracletrace app.py --json v2.json --compare v1.json Great for release validation and refactor checks. +## Fail the run on regression + +```bash +oracletrace my_script.py --json baseline.json +oracletrace my_script.py --json current.json --compare baseline.json --fail-on-regression --threshold 25 +``` + +This is useful in CI when you want the run to fail if performance gets worse by more than 25 percent. + ## Lightweight CI check pattern ```bash diff --git a/oracletrace/cli.py b/oracletrace/cli.py index 1326328..0126e7c 100644 --- a/oracletrace/cli.py +++ b/oracletrace/cli.py @@ -28,6 +28,17 @@ def main(): metavar="NUMBER", help="Limits the number of functions shown in the summary table" ) + parser.add_argument( + "--fail-on-regression", + action="store_true", + help="Exit with a non-zero code when regression exceeds threshold.", + ) + parser.add_argument( + "--threshold", + type=float, + default=5.0, + help="Regression threshold percentage used with --fail-on-regression.", + ) args = parser.parse_args() target = args.target @@ -85,6 +96,7 @@ def main(): "avg_time": fn["avg_time"], }) + comparison_result = None # Compare jsons if args.compare: @@ -95,7 +107,14 @@ def main(): with open(args.compare, "r", encoding="utf-8") as f: old_data = json.load(f) - compare_traces(old_data, data) + comparison_result = compare_traces(old_data, data, threshold=args.threshold) + + if args.fail_on_regression and comparison_result["has_regression"]: + print( + f"Build failed: performance regression above {args.threshold:.2f}% detected." + ) + return 2 + return 0 diff --git a/oracletrace/compare.py b/oracletrace/compare.py index 8a38a68..6cbecf3 100644 --- a/oracletrace/compare.py +++ b/oracletrace/compare.py @@ -1,10 +1,12 @@ from rich import print -def compare_traces(old_data, new_data): +def compare_traces(old_data, new_data, threshold=5.0): old_funcs = {f["name"]: f for f in old_data["functions"]} new_funcs = {f["name"]: f for f in new_data["functions"]} + regressions = [] + print("\n[bold cyan]Comparison Results:[/]\n") all_functions = set(old_funcs) | set(new_funcs) @@ -30,10 +32,25 @@ def compare_traces(old_data, new_data): diff = new_time - old_time percent = (diff / old_time) * 100 - color = "red" if percent > 5 else "green" if percent < -5 else "yellow" + color = "red" if percent > threshold else "green" if percent < -threshold else "yellow" print( f"{name}\n" f" total_time: {old_time:.4f}s → {new_time:.4f}s " f"[{color}]({percent:+.2f}%)[/]\n" ) + + if percent > threshold: + regressions.append( + { + "name": name, + "old_time": old_time, + "new_time": new_time, + "percent": percent, + } + ) + + return { + "regressions": regressions, + "has_regression": len(regressions) > 0, + }