diff --git a/probhub/hashing.py b/probhub/hashing.py index 8c37253..cb4dc9b 100644 --- a/probhub/hashing.py +++ b/probhub/hashing.py @@ -25,9 +25,22 @@ def hash_paths(root, paths, *, normalize_lf_suffixes=()): if not full.is_file(): continue rel = full.relative_to(root).as_posix().encode("utf-8") + if full.suffix.lower() not in normalize_lf_suffixes: + expected_size = full.stat().st_size + digest.update(len(rel).to_bytes(4, "big")) + digest.update(rel) + digest.update(expected_size.to_bytes(8, "big")) + observed_size = 0 + with full.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + observed_size += len(chunk) + digest.update(chunk) + if observed_size != expected_size: + raise OSError(f"file changed while hashing: {full}") + existing.append(full) + continue content = full.read_bytes() - if full.suffix.lower() in normalize_lf_suffixes: - content = content.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + content = content.replace(b"\r\n", b"\n").replace(b"\r", b"\n") digest.update(len(rel).to_bytes(4, "big")) digest.update(rel) digest.update(len(content).to_bytes(8, "big")) diff --git a/probhub/judge_qa.py b/probhub/judge_qa.py new file mode 100644 index 0000000..a49a296 --- /dev/null +++ b/probhub/judge_qa.py @@ -0,0 +1,686 @@ +import hashlib +import re +import unicodedata +from pathlib import Path, PurePosixPath + +from .problem_paths import ProblemPathError, resolve_problem_regular_file + + +JUDGE_QA_SCHEMA_VERSION = 1 +MAX_JUDGE_QA_CASES = 128 +MAX_JUDGE_QA_FILES = 256 +MAX_JUDGE_QA_FILE_BYTES = 16 * 1024 * 1024 +MAX_JUDGE_QA_TOTAL_BYTES = 64 * 1024 * 1024 +MAX_JUDGE_QA_ROBUSTNESS_PROBES = 16 +MAX_JUDGE_QA_DIAGNOSTICS = 128 + +_CASE_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") +_TERMINATION_REASON_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$") +_CHECKER_STATUSES = {"AC", "WA"} +_INTERACTOR_STATUSES = {"AC", "WA", "RE", "TLE", "MLE", "OLE"} +_INTERACTOR_BEHAVIORS = {"early-eof", "idle", "output-flood"} +_ROBUSTNESS_PROBES = {"empty", "truncated", "extra-token", "oversized"} +_PATH_REASON_CODES = { + "invalid": "judge_qa_fixture_path_invalid", + "outside": "judge_qa_fixture_path_outside", + "link": "judge_qa_fixture_path_link", + "missing": "judge_qa_fixture_path_missing", + "non_regular": "judge_qa_fixture_path_non_regular", +} + + +def _normalise_judge_type(config): + judge = config.get("judge") + if not isinstance(judge, dict): + return "standard" + value = str(judge.get("type", "standard")).strip().lower() + return "custom" if value == "checker" else value + + +def _fold_id(value): + return unicodedata.normalize("NFC", value).casefold() + + +def _normalise_relative_path(value): + if not isinstance(value, str) or not value.strip(): + return None + return PurePosixPath(value.strip().replace("\\", "/")).as_posix() + + +def _under_prefix(relative, prefix): + path_parts = PurePosixPath(relative).parts + prefix_parts = PurePosixPath(prefix).parts + return path_parts[:len(prefix_parts)] == prefix_parts and len(path_parts) > len(prefix_parts) + + +def judge_fixture_tree_paths(problem_dir): + """Return regular non-link files stored under judge-fixtures/.""" + + return _judge_qa_tree_paths(problem_dir, "judge-fixtures") + + +def _judge_qa_tree_paths(problem_dir, relative_root): + root = Path(problem_dir).resolve() / relative_root + if not root.is_dir() or root.is_symlink(): + return [] + paths = [] + for candidate in root.rglob("*"): + try: + relative = candidate.relative_to(Path(problem_dir).resolve()).as_posix() + resolved = resolve_problem_regular_file(problem_dir, relative) + except (ProblemPathError, ValueError): + continue + paths.append(resolved) + return paths + + +def _hash_fixture_files(files): + digest = hashlib.sha256() + digest.update(b"probhub-judge-qa-fixtures-v1\0") + for item in sorted(files, key=lambda value: value["path"]): + relative = item["path"].encode("utf-8") + expected_size = item["size"] + digest.update(len(relative).to_bytes(4, "big")) + digest.update(relative) + digest.update(expected_size.to_bytes(8, "big")) + observed_size = 0 + with item["absolute"].open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + observed_size += len(chunk) + digest.update(chunk) + after = item["absolute"].stat() + if ( + observed_size != expected_size + or after.st_size != expected_size + or ( + item.get("stat") + and ( + after.st_dev, + after.st_ino, + after.st_mtime_ns, + after.st_size, + ) + != item["stat"] + ) + ): + raise OSError(f"fixture changed while hashing: {item['path']}") + return digest.hexdigest() + + +def inspect_judge_qa(problem_dir, config): + """Parse Judge QA Schema v1 without executing any fixture.""" + + problem_dir = Path(problem_dir).resolve() + judge = config.get("judge") + if not isinstance(judge, dict) or "qa" not in judge: + return { + "configured": False, + "applicable": _normalise_judge_type(config) in {"custom", "interactive"}, + "ok": True, + "schema_version": None, + "judge_type": _normalise_judge_type(config), + "fixture_hash": None, + "cases": [], + "robustness": None, + "files": [], + "stats": {"cases": 0, "files": 0, "total_bytes": 0}, + "diagnostics": [], + } + + judge_type = _normalise_judge_type(config) + diagnostics = [] + resolved_files = {} + parsed_cases = [] + + def add(code, message, **details): + if len(diagnostics) >= MAX_JUDGE_QA_DIAGNOSTICS: + return + if len(diagnostics) == MAX_JUDGE_QA_DIAGNOSTICS - 1: + diagnostics.append({ + "code": "judge_qa_diagnostics_truncated", + "severity": "error", + "message": ( + "Judge QA diagnostics reached the fixed limit of " + f"{MAX_JUDGE_QA_DIAGNOSTICS}" + ), + "limit": MAX_JUDGE_QA_DIAGNOSTICS, + }) + return + diagnostics.append({ + "code": code, + "severity": "error", + "message": message, + **details, + }) + + def unknown_fields(value, allowed, field, *, case_id=None): + if not isinstance(value, dict): + return + for key in sorted(set(value) - set(allowed), key=str): + add( + "judge_qa_unknown_field", + f"{field} contains unsupported field: {key}", + field=f"{field}.{key}", + **({"case_id": case_id} if case_id else {}), + ) + + def register_file(field, path, *, case_id=None): + try: + canonical_path = path.resolve(strict=True) + canonical_relative = canonical_path.relative_to(problem_dir).as_posix() + stat_before = path.stat() + except (OSError, RuntimeError, ValueError): + add( + "judge_qa_fixture_changed", + f"fixture path changed or became unreadable while resolving: {path}", + field=field, + path=str(path), + **({"case_id": case_id} if case_id else {}), + ) + return None + folded = unicodedata.normalize("NFC", canonical_relative).casefold() + previous = resolved_files.get(folded) + if previous and previous["path"] != canonical_relative: + add( + "judge_qa_fixture_path_collision", + "fixture paths collide case-insensitively on Windows: " + f"{previous['path']} and {canonical_relative}", + field=field, + paths=[previous["path"], canonical_relative], + **({"case_id": case_id} if case_id else {}), + ) + return None + if previous: + return canonical_relative + size = stat_before.st_size + resolved_files[folded] = { + "path": canonical_relative, + "absolute": path, + "size": size, + "stat": ( + stat_before.st_dev, + stat_before.st_ino, + stat_before.st_mtime_ns, + stat_before.st_size, + ), + } + if size > MAX_JUDGE_QA_FILE_BYTES: + add( + "judge_qa_fixture_file_too_large", + f"fixture file exceeds {MAX_JUDGE_QA_FILE_BYTES} bytes: {canonical_relative}", + field=field, + path=canonical_relative, + size=size, + limit=MAX_JUDGE_QA_FILE_BYTES, + **({"case_id": case_id} if case_id else {}), + ) + return canonical_relative + + def resolve_file(field, value, *, required_prefix=None, case_id=None): + relative = _normalise_relative_path(value) + try: + path = resolve_problem_regular_file(problem_dir, value) + except ProblemPathError as exc: + code = _PATH_REASON_CODES.get(exc.reason, "judge_qa_fixture_path_invalid") + add( + code, + f"{field} must be a problem-local regular non-link file: {value!r}", + field=field, + path=value, + reason=exc.reason, + **({"case_id": case_id} if case_id else {}), + ) + return None + if required_prefix and (relative is None or not _under_prefix(relative, required_prefix)): + add( + "judge_qa_fixture_path_scope", + f"{field} must stay under {required_prefix}/: {value!r}", + field=field, + path=value, + required_prefix=required_prefix, + **({"case_id": case_id} if case_id else {}), + ) + return None + + return register_file(field, path, case_id=case_id) + + raw_qa = judge.get("qa") + if not isinstance(raw_qa, dict): + add("judge_qa_schema_invalid", "judge.qa must be a mapping", field="judge.qa") + raw_qa = {} + unknown_fields(raw_qa, {"schema_version", "robustness", "cases"}, "judge.qa") + schema_version = raw_qa.get("schema_version") + if type(schema_version) is not int or schema_version != JUDGE_QA_SCHEMA_VERSION: + add( + "judge_qa_schema_version_unsupported", + f"judge.qa.schema_version must be {JUDGE_QA_SCHEMA_VERSION}", + field="judge.qa.schema_version", + actual=schema_version, + ) + if judge_type not in {"custom", "interactive"}: + add( + "judge_qa_judge_type_unsupported", + "judge.qa is only valid for custom or interactive judging", + field="judge.type", + judge_type=judge_type, + ) + + for tree_name in ("judge-fixtures", "code/judge-qa"): + for path in _judge_qa_tree_paths(problem_dir, tree_name): + register_file("judge.qa.fixture_tree", path) + + raw_cases = raw_qa.get("cases") + if not isinstance(raw_cases, list): + add("judge_qa_cases_invalid", "judge.qa.cases must be a list", field="judge.qa.cases") + raw_cases = [] + elif not raw_cases: + add("judge_qa_cases_empty", "judge.qa.cases must not be empty", field="judge.qa.cases") + if len(raw_cases) > MAX_JUDGE_QA_CASES: + add( + "judge_qa_case_limit_exceeded", + f"judge.qa.cases exceeds the fixed limit of {MAX_JUDGE_QA_CASES}", + field="judge.qa.cases", + count=len(raw_cases), + limit=MAX_JUDGE_QA_CASES, + ) + + seen_ids = {} + expected_by_id = {} + for index, raw_case in enumerate(raw_cases[:MAX_JUDGE_QA_CASES]): + field = f"judge.qa.cases[{index}]" + if not isinstance(raw_case, dict): + add("judge_qa_case_invalid", f"{field} must be a mapping", field=field) + continue + allowed = { + "id", "purpose", "case", "input", "jury_answer", "expected", + "contestant_output" if judge_type == "custom" else "contestant", + } + unknown_fields(raw_case, allowed, field) + + case_id = raw_case.get("id") + if not isinstance(case_id, str) or not _CASE_ID_PATTERN.fullmatch(case_id): + add( + "judge_qa_case_id_invalid", + f"{field}.id must match {_CASE_ID_PATTERN.pattern}", + field=f"{field}.id", + actual=case_id, + ) + case_id = None + else: + folded_id = _fold_id(case_id) + if folded_id in seen_ids: + add( + "judge_qa_case_id_duplicate", + f"Judge QA case IDs collide case-insensitively: " + f"{seen_ids[folded_id]} and {case_id}", + field=f"{field}.id", + case_id=case_id, + conflicts_with=seen_ids[folded_id], + ) + else: + seen_ids[folded_id] = case_id + + purpose = raw_case.get("purpose") + if ( + not isinstance(purpose, str) + or not purpose.strip() + or len(purpose) > 256 + or any(ord(character) < 32 for character in purpose) + ): + add( + "judge_qa_purpose_invalid", + f"{field}.purpose is required and must be a non-empty string of at most 256 characters", + field=f"{field}.purpose", + **({"case_id": case_id} if case_id else {}), + ) + + parsed = { + "id": case_id, + "purpose": purpose.strip() if isinstance(purpose, str) else None, + } + formal_case = raw_case.get("case") + explicit_input = raw_case.get("input") + explicit_answer = raw_case.get("jury_answer") + has_formal = formal_case is not None + has_explicit = explicit_input is not None or explicit_answer is not None + if has_formal and has_explicit: + add( + "judge_qa_case_reference_conflict", + f"{field} cannot combine case with input/jury_answer", + field=field, + **({"case_id": case_id} if case_id else {}), + ) + elif has_formal: + reference = _normalise_relative_path(formal_case) + parts = PurePosixPath(reference).parts if reference else () + if ( + len(parts) != 2 + or parts[0] not in {"sample", "secret"} + or parts[1] in {"", ".", ".."} + or parts[1].casefold().endswith((".in", ".ans")) + ): + add( + "judge_qa_case_reference_invalid", + f"{field}.case must be sample/ or secret/ without an extension", + field=f"{field}.case", + actual=formal_case, + **({"case_id": case_id} if case_id else {}), + ) + else: + data = config.get("data") if isinstance(config.get("data"), dict) else {} + directory_key = "sample_dir" if parts[0] == "sample" else "secret_dir" + default = "data/sample" if parts[0] == "sample" else "data/secret" + base = data.get(directory_key, default) + if not isinstance(base, str) or not base.strip(): + add( + "judge_qa_case_reference_invalid", + f"data.{directory_key} must be a non-empty path string", + field=f"data.{directory_key}", + **({"case_id": case_id} if case_id else {}), + ) + else: + base = base.strip().replace("\\", "/").rstrip("/") + input_path = resolve_file( + f"{field}.case.input", + f"{base}/{parts[1]}.in", + case_id=case_id, + ) + answer_path = resolve_file( + f"{field}.case.jury_answer", + f"{base}/{parts[1]}.ans", + case_id=case_id, + ) + parsed.update({ + "reference": reference, + "input": input_path, + "jury_answer": answer_path, + }) + elif explicit_input is None or explicit_answer is None: + add( + "judge_qa_case_reference_incomplete", + f"{field} must declare case or both input and jury_answer", + field=field, + **({"case_id": case_id} if case_id else {}), + ) + else: + parsed["input"] = resolve_file( + f"{field}.input", + explicit_input, + required_prefix="judge-fixtures", + case_id=case_id, + ) + parsed["jury_answer"] = resolve_file( + f"{field}.jury_answer", + explicit_answer, + required_prefix="judge-fixtures", + case_id=case_id, + ) + + expected = raw_case.get("expected") + allowed_expected = {"status"} if judge_type == "custom" else { + "status", "timeout_kind", "termination_reason", + } + if not isinstance(expected, dict): + add( + "judge_qa_expected_invalid", + f"{field}.expected must be a mapping", + field=f"{field}.expected", + **({"case_id": case_id} if case_id else {}), + ) + expected = {} + unknown_fields(expected, allowed_expected, f"{field}.expected", case_id=case_id) + status = expected.get("status") + allowed_statuses = _CHECKER_STATUSES if judge_type == "custom" else _INTERACTOR_STATUSES + if not isinstance(status, str) or status not in allowed_statuses: + add( + "judge_qa_expected_status_invalid", + f"{field}.expected.status must be one of: {', '.join(sorted(allowed_statuses))}", + field=f"{field}.expected.status", + actual=status, + **({"case_id": case_id} if case_id else {}), + ) + timeout_kind = expected.get("timeout_kind") + if timeout_kind is not None and ( + not isinstance(timeout_kind, str) or timeout_kind not in {"idle", "total"} + ): + add( + "judge_qa_timeout_kind_invalid", + f"{field}.expected.timeout_kind must be idle or total", + field=f"{field}.expected.timeout_kind", + actual=timeout_kind, + **({"case_id": case_id} if case_id else {}), + ) + if timeout_kind is not None and status != "TLE": + add( + "judge_qa_timeout_kind_without_tle", + f"{field}.expected.timeout_kind requires status: TLE", + field=f"{field}.expected.timeout_kind", + **({"case_id": case_id} if case_id else {}), + ) + termination_reason = expected.get("termination_reason") + if termination_reason is not None and ( + not isinstance(termination_reason, str) + or not _TERMINATION_REASON_PATTERN.fullmatch(termination_reason) + ): + add( + "judge_qa_termination_reason_invalid", + f"{field}.expected.termination_reason must be a stable lowercase token", + field=f"{field}.expected.termination_reason", + actual=termination_reason, + **({"case_id": case_id} if case_id else {}), + ) + parsed["expected"] = { + key: expected[key] + for key in ("status", "timeout_kind", "termination_reason") + if key in expected + } + if case_id: + expected_by_id[_fold_id(case_id)] = status + + if judge_type == "custom": + output = raw_case.get("contestant_output") + if output is None: + add( + "judge_qa_contestant_output_required", + f"{field}.contestant_output is required for Checker QA", + field=f"{field}.contestant_output", + **({"case_id": case_id} if case_id else {}), + ) + else: + parsed["contestant_output"] = resolve_file( + f"{field}.contestant_output", + output, + required_prefix="judge-fixtures", + case_id=case_id, + ) + elif judge_type == "interactive": + contestant = raw_case.get("contestant") + if not isinstance(contestant, dict): + add( + "judge_qa_contestant_invalid", + f"{field}.contestant must be a mapping", + field=f"{field}.contestant", + **({"case_id": case_id} if case_id else {}), + ) + else: + unknown_fields( + contestant, + {"source", "behavior"}, + f"{field}.contestant", + case_id=case_id, + ) + source = contestant.get("source") + behavior = contestant.get("behavior") + if (source is None) == (behavior is None): + add( + "judge_qa_contestant_mode_conflict", + f"{field}.contestant must declare exactly one of source or behavior", + field=f"{field}.contestant", + **({"case_id": case_id} if case_id else {}), + ) + elif source is not None: + parsed["contestant"] = { + "source": resolve_file( + f"{field}.contestant.source", + source, + required_prefix="code/judge-qa", + case_id=case_id, + ), + } + elif ( + not isinstance(behavior, str) + or behavior not in _INTERACTOR_BEHAVIORS + ): + add( + "judge_qa_contestant_behavior_invalid", + f"{field}.contestant.behavior must be one of: " + f"{', '.join(sorted(_INTERACTOR_BEHAVIORS))}", + field=f"{field}.contestant.behavior", + actual=behavior, + **({"case_id": case_id} if case_id else {}), + ) + else: + parsed["contestant"] = {"behavior": behavior} + parsed_cases.append(parsed) + + robustness = None + raw_robustness = raw_qa.get("robustness") + if raw_robustness is not None: + if judge_type != "custom": + add( + "judge_qa_robustness_unsupported", + "judge.qa.robustness is only valid for Checker QA", + field="judge.qa.robustness", + ) + if not isinstance(raw_robustness, dict): + add( + "judge_qa_robustness_invalid", + "judge.qa.robustness must be a mapping", + field="judge.qa.robustness", + ) + else: + unknown_fields( + raw_robustness, + {"baseline", "probes"}, + "judge.qa.robustness", + ) + baseline = raw_robustness.get("baseline") + probes = raw_robustness.get("probes") + if not isinstance(baseline, str) or not _CASE_ID_PATTERN.fullmatch(baseline): + add( + "judge_qa_robustness_baseline_invalid", + "judge.qa.robustness.baseline must be a valid case ID", + field="judge.qa.robustness.baseline", + actual=baseline, + ) + elif _fold_id(baseline) not in seen_ids: + add( + "judge_qa_robustness_baseline_missing", + f"robustness baseline does not reference a configured case: {baseline}", + field="judge.qa.robustness.baseline", + baseline=baseline, + ) + elif expected_by_id.get(_fold_id(baseline)) != "AC": + add( + "judge_qa_robustness_baseline_not_ac", + f"robustness baseline must expect AC: {baseline}", + field="judge.qa.robustness.baseline", + baseline=baseline, + ) + if not isinstance(probes, list) or not probes: + add( + "judge_qa_robustness_probes_invalid", + "judge.qa.robustness.probes must be a non-empty list", + field="judge.qa.robustness.probes", + ) + probes = [] + elif len(probes) > MAX_JUDGE_QA_ROBUSTNESS_PROBES: + add( + "judge_qa_robustness_probe_limit_exceeded", + "judge.qa.robustness.probes exceeds the fixed limit of " + f"{MAX_JUDGE_QA_ROBUSTNESS_PROBES}", + field="judge.qa.robustness.probes", + count=len(probes), + limit=MAX_JUDGE_QA_ROBUSTNESS_PROBES, + ) + probes = probes[:MAX_JUDGE_QA_ROBUSTNESS_PROBES] + invalid_probes = sorted({ + str(probe) + for probe in probes + if not isinstance(probe, str) or probe not in _ROBUSTNESS_PROBES + }) + if invalid_probes: + add( + "judge_qa_robustness_probe_invalid", + "unsupported robustness probes: " + ", ".join(invalid_probes), + field="judge.qa.robustness.probes", + probes=invalid_probes, + ) + folded_probes = [ + probe.casefold() for probe in probes if isinstance(probe, str) + ] + seen_probes = set() + duplicate_probes = set() + for probe in folded_probes: + if probe in seen_probes: + duplicate_probes.add(probe) + else: + seen_probes.add(probe) + duplicate_probes = sorted(duplicate_probes) + if duplicate_probes: + add( + "judge_qa_robustness_probe_duplicate", + "duplicate robustness probes: " + ", ".join(duplicate_probes), + field="judge.qa.robustness.probes", + probes=duplicate_probes, + ) + robustness = { + "baseline": baseline, + "probes": list(probes), + } + + total_bytes = sum(item["size"] for item in resolved_files.values()) + if len(resolved_files) > MAX_JUDGE_QA_FILES: + add( + "judge_qa_fixture_file_limit_exceeded", + f"Judge QA references {len(resolved_files)} files; limit is {MAX_JUDGE_QA_FILES}", + count=len(resolved_files), + limit=MAX_JUDGE_QA_FILES, + ) + if total_bytes > MAX_JUDGE_QA_TOTAL_BYTES: + add( + "judge_qa_fixture_total_bytes_exceeded", + f"Judge QA fixture bytes exceed {MAX_JUDGE_QA_TOTAL_BYTES}", + total_bytes=total_bytes, + limit=MAX_JUDGE_QA_TOTAL_BYTES, + ) + + fixture_hash = None + if not diagnostics: + try: + fixture_hash = _hash_fixture_files(resolved_files.values()) + except OSError as exc: + add( + "judge_qa_fixture_changed", + f"fixture files changed or became unreadable while hashing: {exc}", + ) + return { + "configured": True, + "applicable": judge_type in {"custom", "interactive"}, + "ok": not diagnostics, + "schema_version": schema_version, + "judge_type": judge_type, + "fixture_hash": fixture_hash, + "cases": parsed_cases, + "robustness": robustness, + "files": sorted( + ({"path": item["path"], "size": item["size"]} for item in resolved_files.values()), + key=lambda item: item["path"], + ), + "stats": { + "cases": len(raw_cases), + "files": len(resolved_files), + "total_bytes": total_bytes, + }, + "diagnostics": diagnostics, + } diff --git a/probhub/linting.py b/probhub/linting.py index 28136a4..8bebd3a 100644 --- a/probhub/linting.py +++ b/probhub/linting.py @@ -15,6 +15,7 @@ from .datagen import recipe_coverage, resolve_data_dir from .errors import ProbHubError from .hashing import files_under, hash_file, hash_paths +from .judge_qa import inspect_judge_qa, judge_fixture_tree_paths from .metadata import build_meta, normalize_display_name from .problem_paths import ProblemPathError, resolve_problem_regular_file from .solutions import analyze_solution_verification @@ -55,6 +56,7 @@ "__pycache__", "code", "data", + "judge-fixtures", "output_validators", } @@ -163,6 +165,7 @@ def problem_source_paths(problem_dir, config): ) and path.suffix.lower() not in CODE_HASH_IGNORED_SUFFIXES ) + paths.extend(judge_fixture_tree_paths(problem_dir)) paths.extend(problem_statement_asset_paths(problem_dir)) return paths @@ -396,6 +399,11 @@ def lint_problem(root, workspace, entry): errors.append("judge.checker requires judge.type: custom") if interactor: errors.append("judge.interactor requires judge.type: interactive") + judge_qa = inspect_judge_qa(problem_dir, config) + errors.extend( + f"[{diagnostic['code']}] {diagnostic['message']}" + for diagnostic in judge_qa["diagnostics"] + ) solutions = config.get("solutions") or {} configured_programs = set() if not isinstance(solutions, dict): @@ -600,6 +608,7 @@ def lint_problem(root, workspace, entry): *calibration["diagnostics"], *constraint_reconciliation["diagnostics"], *(solution_verification.get("diagnostics") or []), + *judge_qa["diagnostics"], ] return { "id": entry["id"], @@ -611,6 +620,7 @@ def lint_problem(root, workspace, entry): "constraint_reconciliation": constraint_reconciliation, "solution_verification": solution_verification, "calibration": calibration, + "judge_qa": judge_qa, "source_hash": source_hash, "data_hash": data_hash, } diff --git a/tests/fixture_support.py b/tests/fixture_support.py index 08793b9..80c30d9 100644 --- a/tests/fixture_support.py +++ b/tests/fixture_support.py @@ -13,7 +13,15 @@ FIXTURE_ROOT = Path(__file__).resolve().parent / "fixtures" -WORKSPACE_FIXTURES = ("standard", "custom", "float", "interactive", "stress") +JUDGE_QA_FIXTURES = ("checker-qa", "interactor-qa") +WORKSPACE_FIXTURES = ( + "standard", + "custom", + "float", + "interactive", + "stress", + *JUDGE_QA_FIXTURES, +) @dataclass(frozen=True) diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index a778dbb..2b5a991 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -5,9 +5,9 @@ projects. Tests must copy a workspace to a temporary directory before running Core commands because Judge, stress, and status checks create local artifacts. `workspaces/` contains independent examples for standard, custom, float, -interactive, and stress workflows. `faults/` contains source files that the -test helper overlays onto a copied workspace to exercise infrastructure and -resource failures. +interactive, stress, Checker QA Schema, and Interactor QA Schema workflows. +`faults/` contains source files that the test helper overlays onto a copied +workspace to exercise infrastructure and resource failures. Do not commit generated binaries, caches, stress counterexamples, PDFs, ZIPs, metadata, or Build Manifests under this directory. diff --git a/tests/fixtures/workspaces/checker-qa/.probhub/workspace.yaml b/tests/fixtures/workspaces/checker-qa/.probhub/workspace.yaml new file mode 100644 index 0000000..2e97118 --- /dev/null +++ b/tests/fixtures/workspaces/checker-qa/.probhub/workspace.yaml @@ -0,0 +1,4 @@ +schema_version: 1 +problems: +- id: F06 + directory: F06 diff --git a/tests/fixtures/workspaces/checker-qa/F06/code/checker.cpp b/tests/fixtures/workspaces/checker-qa/F06/code/checker.cpp new file mode 100644 index 0000000..e760747 --- /dev/null +++ b/tests/fixtures/workspaces/checker-qa/F06/code/checker.cpp @@ -0,0 +1,13 @@ +#include "testlib.h" +#include + +int main(int argc, char** argv) { + registerTestlibCmd(argc, argv); + long long expected = ans.readLong(); + long long actual = ouf.readLong(); + if (!ouf.seekEof()) + quitf(_wa, "extra output"); + if (std::llabs(actual) == std::llabs(expected)) + quitf(_ok, "accepted absolute value"); + quitf(_wa, "wrong absolute value"); +} diff --git a/tests/fixtures/workspaces/checker-qa/F06/code/std.cpp b/tests/fixtures/workspaces/checker-qa/F06/code/std.cpp new file mode 100644 index 0000000..a5c7735 --- /dev/null +++ b/tests/fixtures/workspaces/checker-qa/F06/code/std.cpp @@ -0,0 +1,7 @@ +#include + +int main() { + long long x; + std::cin >> x; + std::cout << x << '\n'; +} diff --git a/tests/fixtures/workspaces/checker-qa/F06/code/validator.cpp b/tests/fixtures/workspaces/checker-qa/F06/code/validator.cpp new file mode 100644 index 0000000..55f93a6 --- /dev/null +++ b/tests/fixtures/workspaces/checker-qa/F06/code/validator.cpp @@ -0,0 +1,8 @@ +#include "testlib.h" + +int main(int argc, char** argv) { + registerValidation(argc, argv); + inf.readInt(1, 100, "x"); + inf.readEoln(); + inf.readEof(); +} diff --git a/tests/fixtures/workspaces/checker-qa/F06/code/wrong.cpp b/tests/fixtures/workspaces/checker-qa/F06/code/wrong.cpp new file mode 100644 index 0000000..a700c28 --- /dev/null +++ b/tests/fixtures/workspaces/checker-qa/F06/code/wrong.cpp @@ -0,0 +1,7 @@ +#include + +int main() { + long long x; + std::cin >> x; + std::cout << 0 << '\n'; +} diff --git a/tests/fixtures/workspaces/checker-qa/F06/data/sample/basic.ans b/tests/fixtures/workspaces/checker-qa/F06/data/sample/basic.ans new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/tests/fixtures/workspaces/checker-qa/F06/data/sample/basic.ans @@ -0,0 +1 @@ +2 diff --git a/tests/fixtures/workspaces/checker-qa/F06/data/sample/basic.in b/tests/fixtures/workspaces/checker-qa/F06/data/sample/basic.in new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/tests/fixtures/workspaces/checker-qa/F06/data/sample/basic.in @@ -0,0 +1 @@ +2 diff --git a/tests/fixtures/workspaces/checker-qa/F06/data/secret/edge.ans b/tests/fixtures/workspaces/checker-qa/F06/data/secret/edge.ans new file mode 100644 index 0000000..7ed6ff8 --- /dev/null +++ b/tests/fixtures/workspaces/checker-qa/F06/data/secret/edge.ans @@ -0,0 +1 @@ +5 diff --git a/tests/fixtures/workspaces/checker-qa/F06/data/secret/edge.in b/tests/fixtures/workspaces/checker-qa/F06/data/secret/edge.in new file mode 100644 index 0000000..7ed6ff8 --- /dev/null +++ b/tests/fixtures/workspaces/checker-qa/F06/data/secret/edge.in @@ -0,0 +1 @@ +5 diff --git a/tests/fixtures/workspaces/checker-qa/F06/judge-fixtures/checker/alternative.out b/tests/fixtures/workspaces/checker-qa/F06/judge-fixtures/checker/alternative.out new file mode 100644 index 0000000..3fbedf6 --- /dev/null +++ b/tests/fixtures/workspaces/checker-qa/F06/judge-fixtures/checker/alternative.out @@ -0,0 +1 @@ +-2 diff --git a/tests/fixtures/workspaces/checker-qa/F06/judge-fixtures/checker/extra.ans b/tests/fixtures/workspaces/checker-qa/F06/judge-fixtures/checker/extra.ans new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/tests/fixtures/workspaces/checker-qa/F06/judge-fixtures/checker/extra.ans @@ -0,0 +1 @@ +3 diff --git a/tests/fixtures/workspaces/checker-qa/F06/judge-fixtures/checker/extra.in b/tests/fixtures/workspaces/checker-qa/F06/judge-fixtures/checker/extra.in new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/tests/fixtures/workspaces/checker-qa/F06/judge-fixtures/checker/extra.in @@ -0,0 +1 @@ +3 diff --git a/tests/fixtures/workspaces/checker-qa/F06/judge-fixtures/checker/extra.out b/tests/fixtures/workspaces/checker-qa/F06/judge-fixtures/checker/extra.out new file mode 100644 index 0000000..5c91fc4 --- /dev/null +++ b/tests/fixtures/workspaces/checker-qa/F06/judge-fixtures/checker/extra.out @@ -0,0 +1 @@ +3 0 diff --git a/tests/fixtures/workspaces/checker-qa/F06/probhub.yaml b/tests/fixtures/workspaces/checker-qa/F06/probhub.yaml new file mode 100644 index 0000000..99b2a9b --- /dev/null +++ b/tests/fixtures/workspaces/checker-qa/F06/probhub.yaml @@ -0,0 +1,45 @@ +schema_version: 1 +id: F06 +name: Checker QA Fixture +limits: + time: 1 + memory: 256 + output: 1 + processes: 8 +statement: + source: problem.md +judge: + type: custom + validator: code/validator.cpp + checker: code/checker.cpp + qa: + schema_version: 1 + robustness: + baseline: accepts-alternative + probes: [empty, truncated, extra-token, oversized] + cases: + - id: accepts-alternative + purpose: valid-alternative + case: sample/basic + contestant_output: judge-fixtures/checker/alternative.out + expected: {status: AC} + - id: rejects-extra-token + purpose: extra-token + input: judge-fixtures/checker/extra.in + jury_answer: judge-fixtures/checker/extra.ans + contestant_output: judge-fixtures/checker/extra.out + expected: {status: WA} +solutions: + accepted: + - file: code/std.cpp + expected: {status: AC, all: true} + brute: [] + wrong: + - file: code/wrong.cpp + expected: {status: WA, all: true} +data: + sample_dir: data/sample + secret_dir: data/secret + recipes: + - case: edge + manual: true diff --git a/tests/fixtures/workspaces/checker-qa/F06/problem.md b/tests/fixtures/workspaces/checker-qa/F06/problem.md new file mode 100644 index 0000000..a1be202 --- /dev/null +++ b/tests/fixtures/workspaces/checker-qa/F06/problem.md @@ -0,0 +1,13 @@ +# Checker QA Fixture + +## 题目描述 + +给定一个非零整数 $x$，输出任意一个绝对值等于 $|x|$ 的整数。 + +## 输入格式 + +输入一个整数 $x$（$1\le x\le 100$）。 + +## 输出格式 + +输出一个满足要求的整数。 diff --git a/tests/fixtures/workspaces/interactor-qa/.probhub/workspace.yaml b/tests/fixtures/workspaces/interactor-qa/.probhub/workspace.yaml new file mode 100644 index 0000000..4a65fa3 --- /dev/null +++ b/tests/fixtures/workspaces/interactor-qa/.probhub/workspace.yaml @@ -0,0 +1,4 @@ +schema_version: 1 +problems: +- id: F07 + directory: F07 diff --git a/tests/fixtures/workspaces/interactor-qa/F07/code/interactor.cpp b/tests/fixtures/workspaces/interactor-qa/F07/code/interactor.cpp new file mode 100644 index 0000000..3e1d05d --- /dev/null +++ b/tests/fixtures/workspaces/interactor-qa/F07/code/interactor.cpp @@ -0,0 +1,12 @@ +#include "testlib.h" +#include + +int main(int argc, char** argv) { + registerInteraction(argc, argv); + long long secret = inf.readLong(); + std::cout << secret << std::endl; + long long actual = ouf.readLong(); + if (actual == 2 * secret) + quitf(_ok, "accepted"); + quitf(_wa, "wrong response"); +} diff --git a/tests/fixtures/workspaces/interactor-qa/F07/code/judge-qa/normal.cpp b/tests/fixtures/workspaces/interactor-qa/F07/code/judge-qa/normal.cpp new file mode 100644 index 0000000..604a868 --- /dev/null +++ b/tests/fixtures/workspaces/interactor-qa/F07/code/judge-qa/normal.cpp @@ -0,0 +1,8 @@ +#include + +int main() { + long long x; + if (!(std::cin >> x)) + return 1; + std::cout << 2 * x << std::endl; +} diff --git a/tests/fixtures/workspaces/interactor-qa/F07/code/std.cpp b/tests/fixtures/workspaces/interactor-qa/F07/code/std.cpp new file mode 100644 index 0000000..604a868 --- /dev/null +++ b/tests/fixtures/workspaces/interactor-qa/F07/code/std.cpp @@ -0,0 +1,8 @@ +#include + +int main() { + long long x; + if (!(std::cin >> x)) + return 1; + std::cout << 2 * x << std::endl; +} diff --git a/tests/fixtures/workspaces/interactor-qa/F07/code/validator.cpp b/tests/fixtures/workspaces/interactor-qa/F07/code/validator.cpp new file mode 100644 index 0000000..55f93a6 --- /dev/null +++ b/tests/fixtures/workspaces/interactor-qa/F07/code/validator.cpp @@ -0,0 +1,8 @@ +#include "testlib.h" + +int main(int argc, char** argv) { + registerValidation(argc, argv); + inf.readInt(1, 100, "x"); + inf.readEoln(); + inf.readEof(); +} diff --git a/tests/fixtures/workspaces/interactor-qa/F07/code/wrong.cpp b/tests/fixtures/workspaces/interactor-qa/F07/code/wrong.cpp new file mode 100644 index 0000000..580ad79 --- /dev/null +++ b/tests/fixtures/workspaces/interactor-qa/F07/code/wrong.cpp @@ -0,0 +1,8 @@ +#include + +int main() { + long long x; + if (!(std::cin >> x)) + return 1; + std::cout << x << std::endl; +} diff --git a/tests/fixtures/workspaces/interactor-qa/F07/data/sample/basic.ans b/tests/fixtures/workspaces/interactor-qa/F07/data/sample/basic.ans new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/fixtures/workspaces/interactor-qa/F07/data/sample/basic.ans @@ -0,0 +1 @@ +0 diff --git a/tests/fixtures/workspaces/interactor-qa/F07/data/sample/basic.in b/tests/fixtures/workspaces/interactor-qa/F07/data/sample/basic.in new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/tests/fixtures/workspaces/interactor-qa/F07/data/sample/basic.in @@ -0,0 +1 @@ +2 diff --git a/tests/fixtures/workspaces/interactor-qa/F07/data/secret/edge.ans b/tests/fixtures/workspaces/interactor-qa/F07/data/secret/edge.ans new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/fixtures/workspaces/interactor-qa/F07/data/secret/edge.ans @@ -0,0 +1 @@ +0 diff --git a/tests/fixtures/workspaces/interactor-qa/F07/data/secret/edge.in b/tests/fixtures/workspaces/interactor-qa/F07/data/secret/edge.in new file mode 100644 index 0000000..7f8f011 --- /dev/null +++ b/tests/fixtures/workspaces/interactor-qa/F07/data/secret/edge.in @@ -0,0 +1 @@ +7 diff --git a/tests/fixtures/workspaces/interactor-qa/F07/probhub.yaml b/tests/fixtures/workspaces/interactor-qa/F07/probhub.yaml new file mode 100644 index 0000000..5c0f360 --- /dev/null +++ b/tests/fixtures/workspaces/interactor-qa/F07/probhub.yaml @@ -0,0 +1,54 @@ +schema_version: 1 +id: F07 +name: Interactor QA Fixture +limits: + time: 2 + memory: 256 + output: 1 + processes: 8 +statement: + source: problem.md +judge: + type: interactive + validator: code/validator.cpp + interactor: code/interactor.cpp + interactive: + idle_limit: 0.5 + transcript_limit: 4096 + qa: + schema_version: 1 + cases: + - id: normal-protocol + purpose: normal + case: secret/edge + contestant: {source: code/judge-qa/normal.cpp} + expected: {status: AC} + - id: early-eof-player + purpose: early-eof + case: sample/basic + contestant: {behavior: early-eof} + expected: {status: WA} + - id: idle-player + purpose: idle + case: sample/basic + contestant: {behavior: idle} + expected: {status: TLE, timeout_kind: idle, termination_reason: time_limit} + - id: output-flood-player + purpose: output-flood + case: sample/basic + contestant: {behavior: output-flood} + expected: {status: OLE, termination_reason: output_limit} +solutions: + accepted: + - file: code/std.cpp + expected: {status: AC, all: true} + brute: [] + wrong: + - file: code/wrong.cpp + expected: {status: WA, all: true} +data: + sample_dir: data/sample + secret_dir: data/secret + recipes: + - case: edge + manual: true diff --git a/tests/fixtures/workspaces/interactor-qa/F07/problem.md b/tests/fixtures/workspaces/interactor-qa/F07/problem.md new file mode 100644 index 0000000..868af82 --- /dev/null +++ b/tests/fixtures/workspaces/interactor-qa/F07/problem.md @@ -0,0 +1,13 @@ +# Interactor QA Fixture + +## 题目描述 + +交互器会给出一个整数 $x$。请立即输出 $2x$ 并刷新输出缓冲区。 + +## 输入格式 + +交互器给出的整数满足 $1\le x\le 100$。 + +## 输出格式 + +输出整数 $2x$ 并刷新输出缓冲区。 diff --git a/tests/test_judge_qa_schema.py b/tests/test_judge_qa_schema.py new file mode 100644 index 0000000..bc9e5cc --- /dev/null +++ b/tests/test_judge_qa_schema.py @@ -0,0 +1,439 @@ +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch +from zipfile import ZipFile +from unittest.mock import Mock + +from probhub.building import write_manifest +from probhub.generations import create_problem_checkpoint +from probhub.io import read_yaml, write_yaml +from probhub.judge_qa import ( + MAX_JUDGE_QA_CASES, + MAX_JUDGE_QA_DIAGNOSTICS, + MAX_JUDGE_QA_FILES, + MAX_JUDGE_QA_FILE_BYTES, + MAX_JUDGE_QA_ROBUSTNESS_PROBES, + MAX_JUDGE_QA_TOTAL_BYTES, + inspect_judge_qa, +) +from probhub.linting import ( + compute_collection_hash, + compute_data_hash, + compute_source_hash, + compute_workspace_hash, + lint_workspace, +) +from probhub.metadata import build_meta +from probhub.package_tools import ( + build_package, + generate_domjudge_config, + prepare_output_validator, +) +from probhub.workspace import load_problem, load_workspace, problem_entries +from tests.fixture_support import copy_workspace_fixture + + +class JudgeQASchemaTests(unittest.TestCase): + def copy_fixture(self, name): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + fixture = copy_workspace_fixture(name, temporary.name) + root, workspace = load_workspace(fixture.root) + entry = problem_entries(workspace)[0] + problem, config = load_problem(root, entry) + return fixture, root, workspace, entry, problem, config + + @staticmethod + def diagnostic_codes(report): + return {diagnostic["code"] for diagnostic in report["diagnostics"]} + + def inspect_config(self, name, mutate): + _, _, _, _, problem, config = self.copy_fixture(name) + mutate(config) + return inspect_judge_qa(problem, config) + + def test_checker_schema_resolves_formal_and_explicit_files(self): + _, root, workspace, _, _, _ = self.copy_fixture("checker-qa") + result = lint_workspace(root, workspace) + self.assertTrue(result["ok"], result) + report = result["problems"][0]["judge_qa"] + self.assertTrue(report["configured"]) + self.assertTrue(report["applicable"]) + self.assertTrue(report["ok"], report) + self.assertEqual(report["schema_version"], 1) + self.assertEqual(report["judge_type"], "custom") + self.assertRegex(report["fixture_hash"], r"^[0-9a-f]{64}$") + self.assertEqual(report["stats"]["cases"], 2) + self.assertEqual(report["stats"]["files"], 6) + self.assertEqual( + {item["path"] for item in report["files"]}, + { + "data/sample/basic.in", + "data/sample/basic.ans", + "judge-fixtures/checker/alternative.out", + "judge-fixtures/checker/extra.in", + "judge-fixtures/checker/extra.ans", + "judge-fixtures/checker/extra.out", + }, + ) + + def test_schema_limits_and_fixture_hash_vectors_are_stable(self): + self.assertEqual(MAX_JUDGE_QA_CASES, 128) + self.assertEqual(MAX_JUDGE_QA_FILES, 256) + self.assertEqual(MAX_JUDGE_QA_FILE_BYTES, 16 * 1024 * 1024) + self.assertEqual(MAX_JUDGE_QA_TOTAL_BYTES, 64 * 1024 * 1024) + self.assertEqual(MAX_JUDGE_QA_ROBUSTNESS_PROBES, 16) + self.assertEqual(MAX_JUDGE_QA_DIAGNOSTICS, 128) + expected = { + "checker-qa": "7b5c879af93ad156a8d248ed1ee08be1db43fa119b6612f59d4f7bee5e646d41", + "interactor-qa": "54957c98a1a4b9584489402a54c133430f746633a040124ee8970b2578e4c3c2", + } + for name, digest in expected.items(): + with self.subTest(name=name): + _, _, _, _, problem, config = self.copy_fixture(name) + self.assertEqual(inspect_judge_qa(problem, config)["fixture_hash"], digest) + + def test_diagnostic_and_robustness_probe_limits_are_bounded(self): + _, _, _, _, problem, config = self.copy_fixture("checker-qa") + config["judge"]["qa"]["robustness"]["probes"] = ["empty"] * 17 + report = inspect_judge_qa(problem, config) + self.assertIn( + "judge_qa_robustness_probe_limit_exceeded", + self.diagnostic_codes(report), + ) + + _, _, _, _, problem, config = self.copy_fixture("checker-qa") + for index in range(200): + config["judge"]["qa"][f"unknown_{index}"] = True + report = inspect_judge_qa(problem, config) + self.assertEqual(len(report["diagnostics"]), MAX_JUDGE_QA_DIAGNOSTICS) + self.assertEqual(report["diagnostics"][-1]["code"], "judge_qa_diagnostics_truncated") + + def test_interactor_schema_resolves_source_and_builtin_behaviors(self): + _, root, workspace, _, _, _ = self.copy_fixture("interactor-qa") + result = lint_workspace(root, workspace) + self.assertTrue(result["ok"], result) + report = result["problems"][0]["judge_qa"] + self.assertTrue(report["ok"], report) + self.assertEqual(report["judge_type"], "interactive") + self.assertEqual(report["stats"]["cases"], 4) + self.assertEqual(report["stats"]["files"], 5) + contestants = [case.get("contestant") for case in report["cases"]] + self.assertIn({"source": "code/judge-qa/normal.cpp"}, contestants) + self.assertIn({"behavior": "early-eof"}, contestants) + self.assertIn({"behavior": "idle"}, contestants) + self.assertIn({"behavior": "output-flood"}, contestants) + + def test_missing_qa_is_compatible_and_not_reported_as_passed(self): + _, root, workspace, _, _, _ = self.copy_fixture("custom") + result = lint_workspace(root, workspace) + self.assertTrue(result["ok"], result) + report = result["problems"][0]["judge_qa"] + self.assertFalse(report["configured"]) + self.assertTrue(report["applicable"]) + self.assertTrue(report["ok"]) + self.assertIsNone(report["fixture_hash"]) + self.assertEqual(report["diagnostics"], []) + + def test_lint_exposes_stable_judge_qa_diagnostic_codes(self): + fixture, root, workspace, _, _, config = self.copy_fixture("checker-qa") + config["judge"]["qa"]["schema_version"] = 2 + write_yaml(fixture.config_path, config) + result = lint_workspace(root, workspace) + self.assertFalse(result["ok"], result) + problem_result = result["problems"][0] + codes = {item["code"] for item in problem_result["diagnostics"]} + self.assertIn("judge_qa_schema_version_unsupported", codes) + self.assertTrue(any( + error.startswith("[judge_qa_schema_version_unsupported]") + for error in problem_result["errors"] + )) + + def test_schema_and_mutual_exclusion_diagnostics_are_stable(self): + mutations = { + "judge_qa_schema_version_unsupported": ( + "checker-qa", + lambda config: config["judge"]["qa"].update(schema_version=True), + ), + "judge_qa_unknown_field": ( + "checker-qa", + lambda config: config["judge"]["qa"].update(typo=True), + ), + "judge_qa_case_id_duplicate": ( + "checker-qa", + lambda config: config["judge"]["qa"]["cases"][1].update( + id="ACCEPTS-ALTERNATIVE" + ), + ), + "judge_qa_case_reference_conflict": ( + "checker-qa", + lambda config: config["judge"]["qa"]["cases"][1].update( + case="sample/basic" + ), + ), + "judge_qa_case_reference_incomplete": ( + "checker-qa", + lambda config: config["judge"]["qa"]["cases"][1].pop("jury_answer"), + ), + "judge_qa_expected_status_invalid_type": ( + "checker-qa", + lambda config: config["judge"]["qa"]["cases"][0]["expected"].update( + status="FAIL" + ), + ), + "judge_qa_expected_status_invalid": ( + "checker-qa", + lambda config: config["judge"]["qa"]["cases"][0]["expected"].update( + status=[] + ), + ), + "judge_qa_purpose_invalid": ( + "checker-qa", + lambda config: config["judge"]["qa"]["cases"][0].pop("purpose"), + ), + "judge_qa_judge_type_unsupported": ( + "checker-qa", + lambda config: config["judge"].update(type="standard"), + ), + "judge_qa_contestant_mode_conflict": ( + "interactor-qa", + lambda config: config["judge"]["qa"]["cases"][0]["contestant"].update( + behavior="idle" + ), + ), + "judge_qa_timeout_kind_without_tle": ( + "interactor-qa", + lambda config: config["judge"]["qa"]["cases"][0]["expected"].update( + timeout_kind="idle" + ), + ), + "judge_qa_timeout_kind_invalid_type": ( + "interactor-qa", + lambda config: config["judge"]["qa"]["cases"][0]["expected"].update( + timeout_kind=[] + ), + ), + "judge_qa_contestant_behavior_invalid_type": ( + "interactor-qa", + lambda config: config["judge"]["qa"]["cases"][1]["contestant"].update( + behavior=[] + ), + ), + } + for code, (name, mutate) in mutations.items(): + with self.subTest(code=code): + report = self.inspect_config(name, mutate) + self.assertFalse(report["ok"], report) + expected_code = { + "judge_qa_expected_status_invalid_type": "judge_qa_expected_status_invalid", + "judge_qa_timeout_kind_invalid_type": "judge_qa_timeout_kind_invalid", + "judge_qa_contestant_behavior_invalid_type": "judge_qa_contestant_behavior_invalid", + }.get(code, code) + self.assertIn(expected_code, self.diagnostic_codes(report), report) + + def test_resolver_race_returns_stable_diagnostic(self): + _, _, _, _, problem, config = self.copy_fixture("checker-qa") + broken_path = Mock() + broken_path.resolve.side_effect = FileNotFoundError("injected race") + with ( + patch("probhub.judge_qa._judge_qa_tree_paths", return_value=[]), + patch("probhub.judge_qa.resolve_problem_regular_file", return_value=broken_path), + ): + report = inspect_judge_qa(problem, config) + self.assertIn("judge_qa_fixture_changed", self.diagnostic_codes(report), report) + + def test_fixture_paths_reject_missing_outside_directory_and_wrong_scope(self): + cases = { + "judge_qa_fixture_path_invalid": "", + "judge_qa_fixture_path_missing": "judge-fixtures/checker/missing.out", + "judge_qa_fixture_path_outside": "../outside.out", + "judge_qa_fixture_path_non_regular": "judge-fixtures/checker", + "judge_qa_fixture_path_scope": "data/sample/basic.in", + } + for code, value in cases.items(): + with self.subTest(code=code): + report = self.inspect_config( + "checker-qa", + lambda config, value=value: config["judge"]["qa"]["cases"][0].update( + contestant_output=value + ), + ) + self.assertIn(code, self.diagnostic_codes(report), report) + + def test_fixture_paths_reject_links(self): + _, _, _, _, problem, config = self.copy_fixture("checker-qa") + target = problem / "judge-fixtures/checker/alternative.out" + link = problem / "judge-fixtures/checker/linked.out" + try: + os.symlink(target, link) + except (OSError, NotImplementedError) as exc: + self.skipTest(f"file symlinks are unavailable: {exc}") + config["judge"]["qa"]["cases"][0]["contestant_output"] = ( + "judge-fixtures/checker/linked.out" + ) + report = inspect_judge_qa(problem, config) + self.assertIn("judge_qa_fixture_path_link", self.diagnostic_codes(report), report) + + @unittest.skipIf(os.name == "nt", "Windows cannot create case-colliding files") + def test_fixture_paths_reject_windows_case_collisions(self): + _, _, _, _, problem, config = self.copy_fixture("checker-qa") + upper = problem / "judge-fixtures/checker/Output.out" + lower = problem / "judge-fixtures/checker/output.out" + upper.write_bytes(b"2\n") + lower.write_bytes(b"2\n") + cases = config["judge"]["qa"]["cases"] + cases[0]["contestant_output"] = "judge-fixtures/checker/Output.out" + cases[1]["contestant_output"] = "judge-fixtures/checker/output.out" + report = inspect_judge_qa(problem, config) + self.assertIn("judge_qa_fixture_path_collision", self.diagnostic_codes(report), report) + + def test_fixed_case_file_and_byte_limits_are_enforced(self): + _, _, _, _, problem, config = self.copy_fixture("checker-qa") + with patch("probhub.judge_qa.MAX_JUDGE_QA_CASES", 1): + report = inspect_judge_qa(problem, config) + self.assertIn("judge_qa_case_limit_exceeded", self.diagnostic_codes(report), report) + with patch("probhub.judge_qa.MAX_JUDGE_QA_FILES", 5): + report = inspect_judge_qa(problem, config) + self.assertIn( + "judge_qa_fixture_file_limit_exceeded", + self.diagnostic_codes(report), + report, + ) + + baseline = inspect_judge_qa(problem, config) + largest = max(item["size"] for item in baseline["files"]) + total = baseline["stats"]["total_bytes"] + with patch("probhub.judge_qa.MAX_JUDGE_QA_FILE_BYTES", largest): + self.assertNotIn( + "judge_qa_fixture_file_too_large", + self.diagnostic_codes(inspect_judge_qa(problem, config)), + ) + with patch("probhub.judge_qa.MAX_JUDGE_QA_FILE_BYTES", largest - 1): + self.assertIn( + "judge_qa_fixture_file_too_large", + self.diagnostic_codes(inspect_judge_qa(problem, config)), + ) + with patch("probhub.judge_qa.MAX_JUDGE_QA_TOTAL_BYTES", total): + self.assertNotIn( + "judge_qa_fixture_total_bytes_exceeded", + self.diagnostic_codes(inspect_judge_qa(problem, config)), + ) + with patch("probhub.judge_qa.MAX_JUDGE_QA_TOTAL_BYTES", total - 1): + self.assertIn( + "judge_qa_fixture_total_bytes_exceeded", + self.diagnostic_codes(inspect_judge_qa(problem, config)), + ) + + def test_fixture_hash_and_checkpoint_track_exact_fixture_bytes(self): + _, root, workspace, entry, problem, config = self.copy_fixture("checker-qa") + fixture_path = problem / "judge-fixtures/checker/alternative.out" + first_fixture_hash = inspect_judge_qa(problem, config)["fixture_hash"] + first_source_hash = compute_source_hash(problem, config) + first_checkpoint = create_problem_checkpoint(root, workspace, entry) + copied = Path(first_checkpoint["problem_dir"]) / "judge-fixtures/checker/alternative.out" + self.assertEqual(copied.read_bytes(), fixture_path.read_bytes()) + + fixture_path.write_bytes(fixture_path.read_bytes().replace(b"\n", b"\r\n")) + second_fixture_hash = inspect_judge_qa(problem, config)["fixture_hash"] + second_source_hash = compute_source_hash(problem, config) + second_checkpoint = create_problem_checkpoint(root, workspace, entry) + + self.assertNotEqual(first_fixture_hash, second_fixture_hash) + self.assertNotEqual(first_source_hash, second_source_hash) + self.assertNotEqual(first_checkpoint["revision_id"], second_checkpoint["revision_id"]) + copied = Path(second_checkpoint["problem_dir"]) / "judge-fixtures/checker/alternative.out" + self.assertEqual(copied.read_bytes(), b"-2\r\n") + + def test_checkpoint_tracks_interactor_qa_source(self): + _, root, workspace, entry, problem, config = self.copy_fixture("interactor-qa") + source = problem / "code/judge-qa/normal.cpp" + first_hash = compute_source_hash(problem, config) + first = create_problem_checkpoint(root, workspace, entry) + source.write_bytes(source.read_bytes() + b"\n// changed\n") + second_hash = compute_source_hash(problem, config) + second = create_problem_checkpoint(root, workspace, entry) + self.assertNotEqual(first_hash, second_hash) + self.assertNotEqual(first["revision_id"], second["revision_id"]) + copied = Path(second["problem_dir"]) / "code/judge-qa/normal.cpp" + self.assertTrue(copied.read_bytes().endswith(b"// changed\n")) + + def test_unreferenced_fixture_is_tracked_but_not_part_of_pdf_identity(self): + _, root, workspace, entry, problem, config = self.copy_fixture("checker-qa") + first_source_hash = compute_source_hash(problem, config) + first_fixture_hash = inspect_judge_qa(problem, config)["fixture_hash"] + first_collection_hash = compute_collection_hash(root, workspace) + extra = problem / "judge-fixtures/checker/unreferenced.bin" + extra.write_bytes(b"unreferenced fixture\x00") + second_source_hash = compute_source_hash(problem, config) + second_fixture_hash = inspect_judge_qa(problem, config)["fixture_hash"] + second_collection_hash = compute_collection_hash(root, workspace) + checkpoint = create_problem_checkpoint(root, workspace, entry) + copied = Path(checkpoint["problem_dir"]) / "judge-fixtures/checker/unreferenced.bin" + self.assertNotEqual(first_source_hash, second_source_hash) + self.assertNotEqual(first_fixture_hash, second_fixture_hash) + self.assertEqual(first_collection_hash, second_collection_hash) + self.assertEqual(copied.read_bytes(), b"unreferenced fixture\x00") + + def test_qa_files_do_not_enter_collection_package_or_manifest(self): + _, root, workspace, _, problem, config = self.copy_fixture("checker-qa") + first_meta = build_meta(problem, config) + first_collection_hash = compute_collection_hash(root, workspace) + first_source_hash = compute_source_hash(problem, config) + marker = b"fixture-only-sentinel" + fixture_path = problem / "judge-fixtures/checker/alternative.out" + fixture_path.write_bytes(marker) + second_meta = build_meta(problem, config) + second_collection_hash = compute_collection_hash(root, workspace) + second_source_hash = compute_source_hash(problem, config) + self.assertEqual(first_meta, second_meta) + self.assertEqual(first_collection_hash, second_collection_hash) + self.assertNotEqual(first_source_hash, second_source_hash) + + (problem / "problem.pdf").write_bytes(b"%PDF-1.4\nfixture\n") + generate_domjudge_config(problem, config) + prepare_output_validator(problem, config) + package = root / "F06.zip" + build_package(problem, package, config=config) + with ZipFile(package) as archive: + names = archive.namelist() + self.assertFalse(any(name.startswith("judge-fixtures/") for name in names), names) + self.assertFalse(any(name.startswith("code/judge-qa/") for name in names), names) + self.assertNotIn(marker, b"".join(archive.read(name) for name in names)) + + manifest = write_manifest( + problem, + config, + package, + second_source_hash, + compute_data_hash(problem, config), + compute_workspace_hash(root, workspace), + second_collection_hash, + {"digest": "fixture-builder"}, + "fixture-batch", + "fixture-revision", + ) + self.assertNotIn("fixture_hash", manifest) + self.assertNotIn("judge_qa", manifest) + self.assertNotIn("judge-fixtures", str(manifest)) + self.assertNotIn("fixture-only-sentinel", str(manifest)) + + def test_interactor_qa_sources_do_not_enter_package(self): + _, root, _, _, problem, config = self.copy_fixture("interactor-qa") + marker = b"judge-qa-source-only-sentinel" + (problem / "code/judge-qa/extra.cpp").write_bytes(marker) + (problem / "problem.pdf").write_bytes(b"%PDF-1.4\nfixture\n") + generate_domjudge_config(problem, config) + prepare_output_validator(problem, config) + package = root / "F07.zip" + build_package(problem, package, config=config) + with ZipFile(package) as archive: + names = archive.namelist() + self.assertFalse(any(name.startswith("judge-fixtures/") for name in names), names) + self.assertFalse(any(name.startswith("code/judge-qa/") for name in names), names) + self.assertNotIn(marker, b"".join(archive.read(name) for name in names)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_workspace_fixtures.py b/tests/test_workspace_fixtures.py index 20f567b..e325df2 100644 --- a/tests/test_workspace_fixtures.py +++ b/tests/test_workspace_fixtures.py @@ -31,6 +31,7 @@ def test_committed_fixtures_contain_no_generated_artifacts(self): forbidden_names = { "build-manifest.json", "domjudge-problem.ini", + "judge-qa-evidence-v1.json", "meta.json", "problem.pdf", "problem.yaml", @@ -73,6 +74,8 @@ def test_standard_custom_float_and_interactive_judges(self): "custom": "custom", "float": "custom", "interactive": "interactive", + "checker-qa": "custom", + "interactor-qa": "interactive", } for name, judge_type in expected_types.items(): with self.subTest(name=name):