diff --git a/.github/scripts/run_smoke.py b/.github/scripts/run_smoke.py index 11a994a..53859e2 100644 --- a/.github/scripts/run_smoke.py +++ b/.github/scripts/run_smoke.py @@ -1,112 +1,74 @@ """ -Run the workspace smoke test suite. - -Reads `smoke_tests.txt` from the workspace root and `config/build/profile_smoke.yaml` -for per-script env var overrides, then runs each listed script with the -appropriate environment. Continues through failures and exits non-zero -if any script failed. - -The env resolution itself is NOT implemented here: it is PyAutoHands's -`autohands/env_config.py`, imported below. This file used to carry a copy, and -the copy had already drifted (its `load_env_config` hardcoded -`config/build/profile_smoke.yaml`, so the PR gate was structurally unable to read -the release profile — the seed incident's failure mode 4/7). One resolver -means the PR gate and the release runner cannot disagree about what a script's -environment is. See PyAutoHands docs/env_profile_redesign.md §5 (#161 step 2). - -Mirrors the logic of the `/smoke-test` skill so CI and local runs stay -in sync. +Run the workspace smoke test suite: every script under `scripts/`, minus the +exclusions in `config/build/no_run.yaml`. + +Coverage is **opt-out**. A new tutorial is smoke-tested the moment it is added; +excluding one is a deliberate, documented entry in `config/build/no_run.yaml` — +the same file the notebook runner already honours, so scripts and notebooks can +no longer disagree about what is skipped. + +This replaces the former `smoke_tests.txt` allowlist, under which a script was +tested only if someone remembered to add it. That design left HowToGalaxy +testing 4 of its 26 scripts, and a public teaching notebook stayed broken in +three places because no job had ever executed it (HowToGalaxy #58). + +Nothing about discovery, exclusion or environment resolution is implemented +here. This is a thin shim over PyAutoHands' `autohands/run_python.py` — the same +entry point PyAutoHeart's workspace-validation uses for its `run_scripts` job — +so the PR gate and the validation runner cannot drift apart. That runner +provides: + + * recursive discovery, ordering `simulator*` first and then `start_here.py`, + which is what tutorials depending on simulated datasets need + * `should_skip()` against `config/build/no_run.yaml` + * per-script env from `config/build/profile_smoke.yaml` + +Mirrors the `/smoke-test` skill so CI and local runs stay in sync. """ from __future__ import annotations +import os import subprocess import sys -import time from pathlib import Path - WORKSPACE = Path(__file__).resolve().parents[2] -SMOKE_FILE = WORKSPACE / "smoke_tests.txt" -ENV_VARS_FILE = WORKSPACE / "config" / "build" / "profile_smoke.yaml" -SCRIPTS_DIR = WORKSPACE / "scripts" +PROJECT = "howtofit" # CI puts PyAutoHands/autohands on PYTHONPATH (PyAutoHeart's reusable # smoke-tests.yml clones it alongside the dependency chain); for local runs, # fall back to the sibling checkout. try: - from env_config import build_env_for_script, load_env_config + import build_util except ImportError: # pragma: no cover - local-run fallback sys.path.insert(0, str(WORKSPACE.parent / "PyAutoHands" / "autohands")) - from env_config import build_env_for_script, load_env_config - - -def load_smoke_scripts() -> list[str]: - scripts: list[str] = [] - for line in SMOKE_FILE.read_text().splitlines(): - line = line.strip() - if not line or line.startswith("#"): - continue - scripts.append(line) - return scripts - + import build_util -def load_cfg() -> dict | None: - """Parsed env profile, or None when the workspace has none. - - None flows through build_env_for_script -> None -> subprocess inherits the - parent environment, which is what the old local copy's empty-config path - did by hand. - """ - if not ENV_VARS_FILE.exists(): - return None - return load_env_config(ENV_VARS_FILE) - - -def run_one(script_rel: str, cfg: dict | None) -> tuple[str, int, float, str]: - env = build_env_for_script(Path(script_rel), cfg) - script_path = SCRIPTS_DIR / script_rel - t0 = time.time() - result = subprocess.run( - [sys.executable, str(script_path)], - cwd=str(WORKSPACE), - env=env, - capture_output=True, - text=True, - ) - elapsed = time.time() - t0 - output = result.stdout + result.stderr - return script_rel, result.returncode, elapsed, output +AUTOHANDS = Path(build_util.__file__).resolve().parent def main() -> int: - if not SMOKE_FILE.exists(): - print(f"ERROR: no smoke_tests.txt at {SMOKE_FILE}", file=sys.stderr) - return 1 - scripts = load_smoke_scripts() - if not scripts: - print("No smoke test scripts listed.") - return 0 - cfg = load_cfg() - - print(f"Running {len(scripts)} smoke test script(s) from {SMOKE_FILE.name}\n") - failures: list[tuple[str, int, str]] = [] - for script_rel in scripts: - print(f"::group::{script_rel}") - name, rc, elapsed, output = run_one(script_rel, cfg) - print(output, end="") - status = "PASS" if rc == 0 else f"FAIL (exit {rc})" - print(f"\n[{status}] {name} — {elapsed:.1f}s") - print("::endgroup::") - if rc != 0: - failures.append((name, rc, output)) + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join( + p for p in (str(AUTOHANDS), env.get("PYTHONPATH", "")) if p + ) - total = len(scripts) - passed = total - len(failures) - print(f"\n=== Smoke test summary: {passed}/{total} passed ===") - for name, rc, _ in failures: - print(f" FAIL {name} (exit {rc})") - return 0 if not failures else 1 + # --report-dir is REQUIRED, not cosmetic. run_python.py only propagates + # failures (`sys.exit(1)`) when a report was built; without it the suite + # runs to completion and always exits 0 — a vacuously green gate. It also + # switches execute_script from "abort on the first failure" to "record and + # continue", which is the behaviour the old runner had. + cmd = [ + sys.executable, + str(AUTOHANDS / "run_python.py"), + PROJECT, + "scripts", + "--report-dir", + str(WORKSPACE / "test-results"), + ] + # run_python.py resolves config/build/ relative to the cwd. + return subprocess.run(cmd, cwd=str(WORKSPACE), env=env).returncode if __name__ == "__main__": diff --git a/.gitignore b/.gitignore index 32e5ab2..73edf3f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ dataset/ notebooks/plot/ test_report.md test_results/ + +# Structured smoke/validation reports (run_smoke.py --report-dir) +test-results/ diff --git a/AGENTS.md b/AGENTS.md index bce4af0..7993507 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,7 +35,8 @@ and the generic `af.Model` / `af.Collection` API. ## Testing On CI, every PR is gated on Python **3.12 and 3.13** by `smoke_tests.yml` (runs -`python .github/scripts/run_smoke.py`, driven by `smoke_tests.txt` + `config/build/profile_smoke.yaml` — +`python .github/scripts/run_smoke.py`, which runs **every** script under `scripts/` except the +exclusions in `config/build/no_run.yaml`, with per-script env from `config/build/profile_smoke.yaml` — the definition of green), `navigator_check.yml` (PyAutoHands's reusable navigator-catalogue check; see *Notebooks vs Scripts*), and `url_check.yml` (link checking). The smoke and navigator jobs check out **PyAutoHands** as a sibling and run the PyAuto* libraries from the **same-named branch** of each diff --git a/smoke_tests.txt b/smoke_tests.txt deleted file mode 100644 index a9f00ad..0000000 --- a/smoke_tests.txt +++ /dev/null @@ -1,21 +0,0 @@ -# HowToFit smoke test list — one path per line, relative to `scripts/`. -# Consumed by `.github/scripts/run_smoke.py` (CI) and by the `/smoke_test` skill. -# -# Chapter 1 tutorials are fast and representative — they exercise core PyAutoFit -# API (models, analysis, non-linear searches) and, under PYAUTO_TEST_MODE=2, skip -# the sampler entirely so each runs in under a minute. -# -# tutorial_5_results_and_samples is excluded while the interpolator IndexError -# tracked at https://github.com/PyAutoLabs/PyAutoFit/issues/ is unresolved — -# mirrors the autofit_workspace `no_run.yaml` entry for the same tutorial. - -chapter_1_introduction/start_here.py -chapter_1_introduction/tutorial_1_models.py -chapter_1_introduction/tutorial_2_fitting_data.py -chapter_1_introduction/tutorial_3_non_linear_search.py -chapter_1_introduction/tutorial_4_why_modeling_is_hard.py -chapter_3_graphical_models/tutorial_1_individual_models.py -chapter_3_graphical_models/tutorial_2_graphical_model.py -chapter_3_graphical_models/tutorial_3_graphical_benefits.py -chapter_3_graphical_models/tutorial_4_hierachical_models.py -chapter_3_graphical_models/tutorial_5_expectation_propagation.py