From c6cef6d4fccd972d2f7ea4526fe79010213883c5 Mon Sep 17 00:00:00 2001 From: greenthree <1395214327@qq.com> Date: Thu, 6 Aug 2026 14:09:04 +0800 Subject: [PATCH] add Judge QA delivery regression --- scripts/check_clean_install.py | 50 ++++++++++++++++++++++++++++++-- scripts/check_release.py | 3 ++ tests/test_npm_packages.py | 10 +++++++ tests/test_workspace_fixtures.py | 13 +++++++++ 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/scripts/check_clean_install.py b/scripts/check_clean_install.py index a59e14a..0dde246 100644 --- a/scripts/check_clean_install.py +++ b/scripts/check_clean_install.py @@ -115,6 +115,38 @@ def _delivery_identity(workspace, status): } +def _configure_checker_qa(workspace): + """Add a small Judge QA contract to the generated custom problem. + + The files are written into the temporary clean-install workspace rather + than the package, so this exercises the installed Core without adding test + fixtures to the published npm artifact. + """ + problem = Path(workspace) / "E2E" + config_path = problem / "probhub.yaml" + config = config_path.read_text(encoding="utf-8") + marker = " checker: code/checker.cpp\n" + if marker not in config: + raise CleanInstallError("generated custom problem has no checker configuration") + qa = ( + " qa:\n" + " schema_version: 1\n" + " robustness:\n" + " baseline: accepts-sample\n" + " probes: [empty, truncated, extra-token]\n" + " cases:\n" + " - id: accepts-sample\n" + " purpose: valid-alternative\n" + " case: sample/1\n" + " contestant_output: judge-fixtures/checker/accepted.out\n" + " expected: {status: AC}\n" + ) + config_path.write_text(config.replace(marker, marker + qa, 1), encoding="utf-8") + output = problem / "judge-fixtures/checker/accepted.out" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(b"3\n") + + def run_clean_install(): metadata = validate_metadata() npm = shutil.which("npm") @@ -328,7 +360,12 @@ def run_clean_install(): raise CleanInstallError(f"installed WebUI version mismatch: {webui!r}") common = [probhub, "--workspace", workspace, "--json"] - _run_json([*common, "new", "E2E", "--name", "干净安装闭环"], cwd=project, env=env) + _run_json( + [*common, "new", "E2E", "--name", "干净安装闭环", "--judge", "custom"], + cwd=project, + env=env, + ) + _configure_checker_qa(workspace) generated = _run_json([*common, "gen", "E2E", "--apply"], cwd=project, env=env) cases = generated.get("results", []) if not any(item.get("case") == "random01" and not item.get("manual", False) for item in cases): @@ -338,6 +375,12 @@ def run_clean_install(): final = judged["problems"]["E2E"].get("final") or {} if final.get("code") != "all_expectations_met": raise CleanInstallError(f"judge final event mismatch: {final!r}") + judge_qa = _run_json( + [*common, "judge-qa", "E2E", "--no-cache"], cwd=project, env=env + ) + qa_result = judge_qa.get("problems", {}).get("E2E", {}) + if qa_result.get("status") != "passed": + raise CleanInstallError(f"Judge QA did not pass: {judge_qa!r}") sealed = _run_json([*common, "seal", "E2E"], cwd=project, env=env) if sealed.get("checkpoint", {}).get("state") != "sealed": raise CleanInstallError(f"seal did not publish a sealed checkpoint: {sealed!r}") @@ -383,7 +426,10 @@ def run_clean_install(): "version": metadata["version"], "packages": inventories, "documented_install": ["skill-install", "doctor", "ui-check"], - "workflow": ["doctor", "init", "ui-check", "new", "gen", "judge", "seal", "build", "status", "verify-package"], + "workflow": [ + "doctor", "init", "ui-check", "new", "gen", "judge", + "judge-qa", "seal", "build", "status", "verify-package", + ], "rebuild_equivalent": True, } diff --git a/scripts/check_release.py b/scripts/check_release.py index 524fa20..dcb3859 100644 --- a/scripts/check_release.py +++ b/scripts/check_release.py @@ -189,6 +189,8 @@ def validate_pack_inventories(*, dry_run=True, destination=None): "requirements.txt", "bin/init.js", "bin/probhub.js", "bin/python.js", "probhub/__init__.py", "probhub/cli.py", "probhub/install_deps.py", "probhub/install_skill.py", "probhub/process_control.py", + "probhub/judge_qa.py", "probhub/judge_qa_evidence.py", + "probhub/judge_qa_runtime.py", "probhub/webui_runtime.py", "probhub/assets/fonts/NotoSansCJKsc-Regular.otf", "probhub/assets/fonts/OFL.txt", @@ -204,6 +206,7 @@ def validate_pack_inventories(*, dry_run=True, destination=None): "references/problems.typ", "references/usts.png", "references/testlib.h", "references/installation.md", "references/aggregate-limit-derivation.md", + "references/checker-interactor.md", "references/verification-modes.md", } missing = sorted(required_main - main_paths) diff --git a/tests/test_npm_packages.py b/tests/test_npm_packages.py index 617e0fe..4a232c5 100644 --- a/tests/test_npm_packages.py +++ b/tests/test_npm_packages.py @@ -10,6 +10,8 @@ from pathlib import Path from unittest.mock import patch +from scripts.check_release import npm_pack_manifest + ROOT = Path(__file__).resolve().parents[1] @@ -43,6 +45,14 @@ def test_main_and_compatibility_packages_share_exact_version(self): self.assertIn("scripts/webui/**", main["files"]) self.assertIn("!scripts/audit_python_dependencies.py", main["files"]) + def test_judge_qa_runtime_and_contract_are_published(self): + main_files = { + entry["path"] for entry in npm_pack_manifest(dry_run=True)["files"] + } + self.assertIn("probhub/judge_qa_evidence.py", main_files) + self.assertIn("probhub/judge_qa_runtime.py", main_files) + self.assertIn("references/checker-interactor.md", main_files) + def test_both_packages_expose_cli_and_skill_installer(self): expected_bins = { "probhub-skill": "bin/init.js", diff --git a/tests/test_workspace_fixtures.py b/tests/test_workspace_fixtures.py index e325df2..81373cf 100644 --- a/tests/test_workspace_fixtures.py +++ b/tests/test_workspace_fixtures.py @@ -27,6 +27,19 @@ def test_workspaces_are_independent_lintable_schema_v1_projects(self): self.assertEqual(len(result["problems"]), 1) self.assertEqual(result["problems"][0]["errors"], []) + def test_pre_judge_qa_standard_custom_interactive_workspaces_remain_compatible(self): + # These fixtures intentionally predate judge.qa. They must remain + # lintable and explicitly report not-configured rather than passed. + for name in ("standard", "custom", "interactive"): + with self.subTest(name=name), tempfile.TemporaryDirectory() as temp: + fixture = copy_workspace_fixture(name, temp) + root, workspace = load_workspace(fixture.root) + result = lint_workspace(root, workspace) + self.assertTrue(result["ok"], result) + judge_qa = result["problems"][0]["judge_qa"] + self.assertFalse(judge_qa["configured"], judge_qa) + self.assertEqual(judge_qa["evidence"]["state"], "not-configured") + def test_committed_fixtures_contain_no_generated_artifacts(self): forbidden_names = { "build-manifest.json",