diff --git a/heart/checks/workspace_testmode_timing.py b/heart/checks/workspace_testmode_timing.py new file mode 100644 index 0000000..b7b7b38 --- /dev/null +++ b/heart/checks/workspace_testmode_timing.py @@ -0,0 +1,210 @@ +"""heart/checks/workspace_testmode_timing.py — time curated workspace scripts run +in integration TEST MODE, track a rolling baseline, flag regressions. + +Distinct from ``script_timing``, which reads PyAutoBuild ``run_all`` durations +(the release/CI path). This leg runs a curated set of workspace ``start_here`` +scripts with ``PYAUTO_TEST_MODE=2`` + ``PYAUTO_WORKSPACE_SMALL_DATASETS=1`` — the +developer-loop mode named in the original hygiene prompt — and times them. + +OFF-TICK by design: running scripts costs seconds–minutes, which does not fit +the <30s watch-loop budget (``docs/internals.md`` rule 3). Run on a slower +cadence (daily cron / on demand), NOT from ``heart/tick.sh``: + + python -m heart.checks.workspace_testmode_timing + +Scripts run in a SUBPROCESS (rule 4 — Heart never imports the science stack; the +runner is injected so the stdlib-only tests use fixtures). Advisory: surfaced on +the board but never a release gate. + +State: ~/.pyauto-heart/timings-ws-testmode/.json — rolling window per script. +Output: ~/.pyauto-heart/workspace_testmode_timing.json — regression summary. +Classification mirrors script_timing (ratio = latest / median(prior window)). +""" + +from __future__ import annotations + +import json +import os +import re +import statistics +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Callable + +import yaml + +HEART_STATE_DIR = Path( + os.environ.get("HEART_STATE_DIR") + or Path.home() / ".pyauto-heart" +) +HEART_WS_TIMINGS_DIR = HEART_STATE_DIR / "timings-ws-testmode" +HEART_HOME = Path(__file__).resolve().parents[2] +CONFIG_PATH = HEART_HOME / "config" / "repos.yaml" + +# Curated dev-loop scripts (the smoke-subset spirit — do NOT grow this to make +# the signal feel stronger). (workspace_repo, workspace-relative script path). +DEFAULT_SCRIPTS = [ + ("autolens_workspace", "scripts/imaging/start_here.py"), + ("autogalaxy_workspace", "scripts/imaging/start_here.py"), + ("autofit_workspace", "scripts/overview/overview_1_the_basics.py"), +] + +# A runner maps (workspace, rel_path) to the script's wall-clock in seconds, or +# None when it could not be timed (missing / errored / timed out). Injected in tests. +Runner = Callable[[str, str], "float | None"] + + +def load_thresholds() -> tuple[float, float, int]: + yellow, red, window = 1.5, 3.0, 7 + if CONFIG_PATH.is_file(): + cfg = yaml.safe_load(CONFIG_PATH.read_text()) or {} + t = cfg.get("thresholds", {}).get("workspace_testmode_timing", {}) + yellow = float(t.get("yellow_factor", yellow)) + red = float(t.get("red_factor", red)) + window = int(t.get("baseline_window", window)) + return yellow, red, window + + +def default_runner(root: Path, python: str, timeout: float) -> Runner: + """Time ``python