From 57db093d6a18567299d312b6b4c9351745023487 Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:24:19 +0900 Subject: [PATCH 1/3] feat(bench): verifiability categories and the five-tool memory contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit benchmark v2 grows the axis recall-only benchmarks cannot measure: citation-correctness (the surfaced answer must be spelled by a quote whose byte-offset receipt verifies), receipt-coverage (fraction of surfaced claim items carrying a verifying receipt), and supersede-hygiene (a landed update must retire the stale value from the live claim set). the first two are guards that score 1.00 stock; the third is a lever at 0.00 alongside knowledge-update, pointing at lifecycle-driven supersession. memory_contract.MemoryContract implements ditto's five mining-contract tools (save, search, search subjects, fetch by id, search in subject) over the real store, so a shared-contract head-to-head can run both engines on one task set. saves route through the receipt-gated capture loop and return nothing durable when the gate is off — the review gate is never silently bypassed, even inside a harness. reference baseline refreshed to 0.52 +- 0.03 over seeds 1-6 (the category set change is a bench-contract version bump). --- src/vouch/bench.py | 197 ++++++++++++++++++++++++++++++++-- src/vouch/memory_contract.py | 147 +++++++++++++++++++++++++ tests/test_bench.py | 102 +++++++++++++++++- tests/test_memory_contract.py | 89 +++++++++++++++ 4 files changed, 527 insertions(+), 8 deletions(-) create mode 100644 src/vouch/memory_contract.py create mode 100644 tests/test_memory_contract.py diff --git a/src/vouch/bench.py b/src/vouch/bench.py index 7aca5b4d..1955d968 100644 --- a/src/vouch/bench.py +++ b/src/vouch/bench.py @@ -22,6 +22,15 @@ rebuilds the index, and retrieves through ``context.build_context_pack``. A score means vouch-as-shipped retrieved it, under the same review-gate invariants as always. +* **Verifiability axes.** Three categories grade the store's receipts and + lifecycle state, not the pack text: citation-correctness (the surfaced + answer must be spelled by a quote whose byte-offset receipt verifies), + receipt-coverage (fraction of surfaced claim items carrying a verifying + receipt), and supersede-hygiene (once an update landed, the stale value + must not survive as a live claim). Recall-only engines cannot score here + by construction — the axis receipts make measurable. The shared-contract + half of a head-to-head lives in ``memory_contract.MemoryContract``, the + five-tool (Ditto-contract) adapter over the same store. No model, no network, no wall-clock dependence: `vouch bench run --seed 7` gives the same number on every machine, which is what makes scores comparable @@ -30,17 +39,21 @@ Reference baseline (update when retrieval changes; the levers are the zeros): ====================== ===================================================== -run ``vouch bench run --seeds 1,2,3,4,5,6`` @ 2026-07-27 -composite 0.57 ± 0.04 (SE) +run ``vouch bench run --seeds 1,2,3,4,5,6`` @ 2026-07-28 +composite 0.52 ± 0.03 (SE) single-session-recall 1.00 — verbatim receipts + FTS: plain recall is won -multi-session 0.83 +multi-session 0.50 knowledge-update 0.00 — superseded value stays in the pack; needs lifecycle-driven supersession, not reranking (recency reorders, dump-guard still zeroes) -point-in-time 1.00 +point-in-time 0.83 decoy-discrimination 0.00 — same-attribute other-person value outranks injection-resistance 0.83 -abstention 0.33 — cross-person leak under lexical match +abstention 0.00 — cross-person leak under lexical match +citation-correctness 1.00 — guard: the surfaced answer is receipt-quoted +receipt-coverage 1.00 — guard: surfaced claims are receipt-backed +supersede-hygiene 0.00 — stale value stays live; the lifecycle lever + (same root cause as knowledge-update's zero) ====================== ===================================================== For calibration only (different benchmarks, not directly comparable): @@ -57,7 +70,8 @@ from pathlib import Path from typing import Any -from .storage import KBStore +from .models import ClaimStatus +from .storage import ArtifactNotFoundError, KBStore BENCH_ACTOR = "vouch-bench" DEFAULT_BUDGET_CHARS = 2000 @@ -74,6 +88,19 @@ "decoy-discrimination", "injection-resistance", "abstention", + # The verifiability axes — graded against the store's receipts and + # lifecycle state, not the pack text alone. A recall-only benchmark + # (DittoBench included) cannot measure any of these: they require the + # engine to carry byte-offset receipts in the first place. + "citation-correctness", + "receipt-coverage", + "supersede-hygiene", +) + +# A superseded, archived, or redacted claim is not a live memory (mirrors +# the set the context pack excludes). +_RETIRED_STATUSES = frozenset( + (ClaimStatus.SUPERSEDED, ClaimStatus.ARCHIVED, ClaimStatus.REDACTED) ) _DECOY_PERSON = "alice-example" @@ -116,6 +143,18 @@ "what is my parking spot?", "which parking spot is mine?", )), + ("test runner", ( + "which test runner do we use?", + "what runner executes the test suite?", + )), + ("backup cadence", ( + "how often do backups run?", + "what is the backup cadence?", + )), + ("standup time", ( + "when is the daily standup?", + "what time is standup?", + )), ) _STATEMENT_TEMPLATES = ( @@ -201,6 +240,10 @@ def _coin_value(rng: random.Random, attr: str) -> str: return f"spot {rng.randrange(11, 99)}{rng.choice('bcdfg')}" if attr == "staging region": return f"{_coin_word(rng, 2)}-{rng.randrange(2, 9)}" + if attr == "backup cadence": + return f"every {rng.randrange(3, 9)} hours" + if attr == "standup time": + return f"{rng.randrange(8, 12)}:{rng.choice(('05', '15', '35', '45'))}" return _coin_word(rng) @@ -293,6 +336,36 @@ def question(phrasings: tuple[str, ...]) -> str: )) cases.append(MemoryCase("abstention", question(asks), None, (decoy_value,))) + # 8. citation-correctness: stated once, but graded on the receipt — the + # surfaced answer must be provably quoted from the source bytes. + attr, asks = attrs.pop() + value = _coin_value(rng, attr) + docs.add(spot(), statement(attr, value)) + cases.append(MemoryCase("citation-correctness", question(asks), value)) + + # 9. receipt-coverage: every claim item in the answering pack must carry + # a verifying receipt. A guard category: stock scores 1.0, and a change + # that starts surfacing unbacked content pays for it here. + attr, asks = attrs.pop() + value = _coin_value(rng, attr) + docs.add(spot(), statement(attr, value)) + cases.append(MemoryCase("receipt-coverage", question(asks), value)) + + # 10. supersede-hygiene: v1 then an update to v2, graded on the store — + # the stale value must not survive as a live claim while a live claim + # holds the current one. The lifecycle lever knowledge-update's zero + # points at, made a scored axis of its own. + attr, asks = attrs.pop() + v1 = _coin_value(rng, attr) + v2 = _coin_value(rng, attr) + while v2 == v1: + v2 = _coin_value(rng, attr) + early = rng.randrange(sessions - 1) + late = rng.randrange(early + 1, sessions) + docs.add(early, statement(attr, v1)) + docs.add(late, rng.choice(_UPDATE_TEMPLATES).format(attr=attr, value=v2)) + cases.append(MemoryCase("supersede-hygiene", question(asks), v2, (v1,))) + # Filler prose in every session, shuffled placement. for idx in range(sessions): for _ in range(rng.randrange(2, 5)): @@ -334,6 +407,101 @@ def grade_case(case: MemoryCase, pack_text: str) -> tuple[float, str | None]: return 0.0, "expected value not surfaced" +def _verified_quotes(store: KBStore, citation_ids: list[str]) -> list[str]: + """Quotes of the citations whose byte-offset receipts verify. + + A bare source-id citation, a dangling id, or a forged span contributes + nothing — only a receipt that verifies by string comparison counts. + """ + from .receipts import verify_receipt + + quotes: list[str] = [] + for cid in citation_ids: + try: + ev = store.get_evidence(cid) + raw = store.read_source_content(ev.source_id) + except (ArtifactNotFoundError, OSError): + continue + if verify_receipt(ev, raw).verified and ev.quote: + quotes.append(ev.quote) + return quotes + + +def _item_citations(item: dict[str, Any]) -> list[str]: + return [str(c) for c in item.get("citations", [])] + + +def grade_citation_correctness( + store: KBStore, case: MemoryCase, pack: dict[str, Any] +) -> tuple[float, str | None]: + """The answer must be receipt-backed, not merely present. + + Some claim item surfacing the expected value has to carry a citation + whose *verified* quote spells that value — proof the answer was quoted + from real source bytes rather than drifted or fabricated en route. + """ + expected = (case.expected or "").lower() + if expected not in _pack_text(pack).lower(): + return 0.0, "expected value not surfaced" + for item in pack.get("items", []): + if item.get("type") != "claim": + continue + if expected not in str(item.get("summary", "")).lower(): + continue + quotes = _verified_quotes(store, _item_citations(item)) + if any(expected in q.lower() for q in quotes): + return 1.0, None + return 0.0, "answer surfaced without a verifying receipt" + + +def grade_receipt_coverage( + store: KBStore, case: MemoryCase, pack: dict[str, Any] +) -> tuple[float, str | None]: + """Fraction of the pack's claim items backed by a verifying receipt. + + Deliberately not gated on the expected value surfacing — recall misses + are priced by the recall categories. This axis measures only that what + the pack *does* surface is mechanically backed; an empty pack surfaces + nothing unbacked and scores full. + """ + claim_items = [ + i for i in pack.get("items", []) if i.get("type") == "claim" + ] + if not claim_items: + return 1.0, None + backed = sum( + 1 for i in claim_items + if _verified_quotes(store, _item_citations(i)) + ) + if backed == len(claim_items): + return 1.0, None + return ( + round(backed / len(claim_items), 4), + f"{len(claim_items) - backed} of {len(claim_items)} claim items " + "lack a verifying receipt", + ) + + +def grade_supersede_hygiene( + store: KBStore, case: MemoryCase +) -> tuple[float, str | None]: + """Graded on the store, not the pack: after ingest the stale value must + not survive as a live claim while a live claim holds the current one — + the KB tells one truth, enforced by lifecycle state.""" + stale = [f.lower() for f in case.forbidden] + current = (case.expected or "").lower() + live = [ + c for c in store.list_claims() if c.status not in _RETIRED_STATUSES + ] + for claim in live: + text = claim.text.lower() + if any(s in text for s in stale): + return 0.0, f"stale value still live in claim {claim.id!r}" + if not any(current in c.text.lower() for c in live): + return 0.0, "no live claim carries the current value" + return 1.0, None + + def _pack_text(pack: dict[str, Any]) -> str: # build_context_pack returns a ContextPack.model_dump() dict (plus # transport extras); the graded surface is what an agent would read. @@ -351,6 +519,7 @@ def run( workdir: Path | None = None, extra_config: str | None = None, session_gap_seconds: float = 0.0, + strategy: Any = None, ) -> dict[str, Any]: """Generate, ingest through the real pipeline, retrieve, and grade. @@ -367,6 +536,11 @@ def run( timestamps carry the sessions' temporal order — the structure a recency-aware arm ranks by. Zero (the default) keeps runs fast; the generated dataset is identical either way. + + ``strategy`` is an optional pluggable ranking strategy (see + ``vouch.strategy``) — the engine-lane arm. For a competition submission + it is a ``SandboxProxy`` wrapping the untrusted file; the same generated + dataset scored with vs without it isolates the strategy's contribution. """ import tempfile import time as time_mod @@ -396,8 +570,17 @@ def run( for case in dataset.cases: pack = build_context_pack( store, query=case.question, limit=limit, max_chars=budget_chars, + strategy=strategy, ) - score, reason = grade_case(case, _pack_text(dict(pack))) + pack_dict = dict(pack) + if case.category == "citation-correctness": + score, reason = grade_citation_correctness(store, case, pack_dict) + elif case.category == "receipt-coverage": + score, reason = grade_receipt_coverage(store, case, pack_dict) + elif case.category == "supersede-hygiene": + score, reason = grade_supersede_hygiene(store, case) + else: + score, reason = grade_case(case, _pack_text(pack_dict)) per_category[case.category].append(score) if reason is not None: failures.append({ diff --git a/src/vouch/memory_contract.py b/src/vouch/memory_contract.py new file mode 100644 index 00000000..90ca94b4 --- /dev/null +++ b/src/vouch/memory_contract.py @@ -0,0 +1,147 @@ +"""The five-tool memory contract, implemented over a vouch KB. + +Ditto's mining contract (SN118) scores a harness on exactly five tools: +save a memory, search memories, search subjects, fetch by id, and search +within a subject. This module is vouch's side of that shared contract — a +thin adapter over the real store, so a head-to-head can run both engines +on one task set instead of comparing incomparable benchmark numbers. + +Two properties are non-negotiable and differ from a plain memory store: + +* **Saves go through the review gate.** ``save_memory`` runs the same + receipt-gated capture loop production uses (``extract.ingest_source``). + With ``review.auto_approve_on_receipt`` off, a save files a proposal and + returns nothing durable — the gate is never silently bypassed, even + inside a benchmark harness. +* **Subjects are provenance, not labels.** A memory's subject is the title + of the source it cites; there is no free-floating subject field to + drift from the evidence. Subject search and subject-scoped search both + resolve through citations. +""" + +from __future__ import annotations + +from contextlib import suppress +from dataclasses import dataclass + +from .context import search_kb +from .extract import ingest_source +from .models import Claim, ClaimStatus, Evidence +from .receipts import verify_receipt +from .storage import ArtifactNotFoundError, KBStore + +CONTRACT_TOOLS = ( + "save_memory", + "search_memories", + "search_subjects", + "fetch_by_id", + "search_in_subject", +) + +# Mirrors the retracted set the context pack excludes: a superseded, +# archived, or redacted claim is not a live memory. +_RETRACTED = frozenset( + (ClaimStatus.SUPERSEDED, ClaimStatus.ARCHIVED, ClaimStatus.REDACTED) +) + + +@dataclass(frozen=True) +class MemoryHit: + """One memory as the contract returns it.""" + + id: str + text: str + score: float + subject: str | None + receipt_backed: bool + + +class MemoryContract: + """The five contract tools over one vouch KB.""" + + def __init__(self, store: KBStore, *, actor: str = "memory-contract") -> None: + self.store = store + self.actor = actor + + def save_memory(self, text: str, *, subject: str | None = None) -> list[str]: + """Store one memory; return the ids that became durable. + + An empty list means the receipt gate is off and the memory is + pending human review — filed, not durable. + """ + _, approved = ingest_source( + self.store, text.encode("utf-8"), + proposed_by=self.actor, title=subject, + ) + return [claim.id for claim in approved] + + def search_memories(self, query: str, *, limit: int = 10) -> list[MemoryHit]: + result = search_kb(self.store, query=query, limit=limit) + hits: list[MemoryHit] = [] + for row in result["hits"]: + if row["kind"] != "claim": + continue + try: + claim = self.store.get_claim(str(row["id"])) + except ArtifactNotFoundError: + continue + if claim.status in _RETRACTED: + continue + hits.append(self._hit(claim, score=float(row["score"]))) + return hits + + def search_subjects(self, query: str, *, limit: int = 10) -> list[str]: + needle = query.lower() + subjects = { + subject + for claim in self.store.list_claims() + if claim.status not in _RETRACTED + and (subject := self._subject_of(claim)) is not None + and needle in subject.lower() + } + return sorted(subjects)[:limit] + + def fetch_by_id(self, memory_id: str) -> MemoryHit: + return self._hit(self.store.get_claim(memory_id), score=1.0) + + def search_in_subject( + self, subject: str, query: str, *, limit: int = 10 + ) -> list[MemoryHit]: + wide = self.search_memories(query, limit=max(limit * 5, 25)) + return [hit for hit in wide if hit.subject == subject][:limit] + + def _hit(self, claim: Claim, *, score: float) -> MemoryHit: + return MemoryHit( + id=claim.id, + text=claim.text, + score=score, + subject=self._subject_of(claim), + receipt_backed=any( + verify_receipt( + ev, self.store.read_source_content(ev.source_id) + ).verified + for ev in self._evidence_of(claim) + ), + ) + + def _evidence_of(self, claim: Claim) -> list[Evidence]: + out: list[Evidence] = [] + for cid in claim.evidence: + try: + out.append(self.store.get_evidence(cid)) + except ArtifactNotFoundError: + continue # a bare source-id citation carries no receipt + return out + + def _subject_of(self, claim: Claim) -> str | None: + for cid in claim.evidence: + source_id = cid + with suppress(ArtifactNotFoundError): + source_id = self.store.get_evidence(cid).source_id + try: + title = self.store.get_source(source_id).title + except ArtifactNotFoundError: + continue + if title: + return title + return None diff --git a/tests/test_bench.py b/tests/test_bench.py index ca59e98b..676c706e 100644 --- a/tests/test_bench.py +++ b/tests/test_bench.py @@ -3,12 +3,27 @@ from __future__ import annotations import json +from pathlib import Path from click.testing import CliRunner -from vouch import bench +from vouch import bench, health, lifecycle from vouch.bench import CATEGORIES, MemoryCase, generate, grade_case, run, run_seeds from vouch.cli import cli +from vouch.context import build_context_pack +from vouch.extract import ingest_source +from vouch.storage import KBStore + + +def _mini_kb(tmp_path: Path, *sentences: str) -> KBStore: + store = KBStore.init(tmp_path / "kb") + store.config_path.write_text( + "review:\n auto_approve_on_receipt: true\n", encoding="utf-8" + ) + for sentence in sentences: + ingest_source(store, sentence.encode("utf-8"), proposed_by="bench-test") + health.rebuild_index(store) + return store def test_generate_is_deterministic() -> None: @@ -88,3 +103,88 @@ def test_cli_bench_gen_emits_dataset() -> None: assert payload["seed"] == 5 assert len(payload["cases"]) == len(CATEGORIES) assert payload["sessions"] + + +def test_categories_include_verifiability_axes() -> None: + assert "citation-correctness" in CATEGORIES + assert "receipt-coverage" in CATEGORIES + assert "supersede-hygiene" in CATEGORIES + + +def test_grade_citation_correctness_requires_verifying_receipt( + tmp_path: Path, +) -> None: + store = _mini_kb( + tmp_path, "for the record, my favorite editor is zorvex right now." + ) + case = MemoryCase("citation-correctness", "what is my favorite editor?", "zorvex") + pack = dict( + build_context_pack(store, query=case.question, limit=10, max_chars=2000) + ) + assert bench.grade_citation_correctness(store, case, pack)[0] == 1.0 + # the value surfacing WITHOUT a receipt that verifies is worth nothing — + # a claim item citing a bare/unknown id has no mechanical backing + unbacked = { + "items": [ + { + "id": "x", + "type": "claim", + "summary": "my favorite editor is zorvex", + "citations": ["deadbeef"], + } + ] + } + score, reason = bench.grade_citation_correctness(store, case, unbacked) + assert score == 0.0 + assert reason is not None + + +def test_grade_receipt_coverage_full_on_receipt_backed_pack( + tmp_path: Path, +) -> None: + store = _mini_kb(tmp_path, "the project codename is mulopi now.") + case = MemoryCase("receipt-coverage", "what is the project codename?", "mulopi") + pack = dict( + build_context_pack(store, query=case.question, limit=10, max_chars=2000) + ) + assert bench.grade_receipt_coverage(store, case, pack)[0] == 1.0 + unbacked = { + "items": [ + { + "id": "x", + "type": "claim", + "summary": "the codename is mulopi", + "citations": ["deadbeef"], + } + ] + } + assert bench.grade_receipt_coverage(store, case, unbacked)[0] == 0.0 + + +def test_grade_supersede_hygiene_rewards_lifecycle(tmp_path: Path) -> None: + store = _mini_kb( + tmp_path, + "for the record, my deploy day is monday right now.", + "heads up, the deploy day changed to friday this week.", + ) + case = MemoryCase( + "supersede-hygiene", "which day do we deploy?", "friday", ("monday",) + ) + # stock pipeline leaves the stale value live — the lever this category creates + score, reason = bench.grade_supersede_hygiene(store, case) + assert score == 0.0 + assert reason is not None + old = next(c for c in store.list_claims() if "monday" in c.text) + new = next(c for c in store.list_claims() if "friday" in c.text) + lifecycle.supersede(store, old_claim_id=old.id, new_claim_id=new.id, actor="t") + assert bench.grade_supersede_hygiene(store, case)[0] == 1.0 + + +def test_run_verifiability_stock_scores() -> None: + report = run(1, sessions=4) + # guard categories: the receipt path makes these perfect on the stock + # engine; a change that surfaces unbacked content loses points here + assert report["categories"]["receipt-coverage"]["mean"] == 1.0 + assert report["categories"]["citation-correctness"]["mean"] == 1.0 + # lever category: nothing in the no-model ingest path supersedes yet + assert report["categories"]["supersede-hygiene"]["mean"] == 0.0 diff --git a/tests/test_memory_contract.py b/tests/test_memory_contract.py new file mode 100644 index 00000000..7e8c8b3f --- /dev/null +++ b/tests/test_memory_contract.py @@ -0,0 +1,89 @@ +"""The five-tool memory contract: gate-respecting saves, subject-scoped search.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from vouch.memory_contract import CONTRACT_TOOLS, MemoryContract, MemoryHit +from vouch.storage import ArtifactNotFoundError, KBStore + + +def _kb(tmp_path: Path, *, receipt_gate: bool = True) -> KBStore: + # init's starter config already opts into the receipt gate (phase d); + # the gate-off arm must configure it off explicitly. + store = KBStore.init(tmp_path / "kb") + flag = "true" if receipt_gate else "false" + store.config_path.write_text( + f"review:\n auto_approve_on_receipt: {flag}\n", encoding="utf-8" + ) + return store + + +def test_contract_names_the_five_ditto_tools() -> None: + assert CONTRACT_TOOLS == ( + "save_memory", + "search_memories", + "search_subjects", + "fetch_by_id", + "search_in_subject", + ) + + +def test_save_then_search_roundtrip(tmp_path: Path) -> None: + contract = MemoryContract(_kb(tmp_path)) + saved = contract.save_memory( + "for the record, my favorite editor is zorvex right now." + ) + assert saved, "receipt-gated save produced no durable memory" + hits = contract.search_memories("favorite editor") + assert any("zorvex" in h.text for h in hits) + + +def test_save_without_receipt_gate_stays_pending(tmp_path: Path) -> None: + # The review gate is never silently bypassed: with auto-approve off the + # save files a proposal and nothing becomes durable or searchable. + contract = MemoryContract(_kb(tmp_path, receipt_gate=False)) + saved = contract.save_memory("the staging region is vora-3 as of today.") + assert saved == [] + assert contract.search_memories("staging region") == [] + + +def test_fetch_by_id_returns_saved_memory(tmp_path: Path) -> None: + contract = MemoryContract(_kb(tmp_path)) + saved = contract.save_memory("the project codename is mulopi now.") + hit = contract.fetch_by_id(saved[0]) + assert isinstance(hit, MemoryHit) + assert hit.id == saved[0] + assert "mulopi" in hit.text + assert hit.receipt_backed is True + + +def test_fetch_by_id_unknown_raises(tmp_path: Path) -> None: + contract = MemoryContract(_kb(tmp_path)) + with pytest.raises(ArtifactNotFoundError): + contract.fetch_by_id("no-such-memory") + + +def test_search_subjects_matches_saved_subjects(tmp_path: Path) -> None: + contract = MemoryContract(_kb(tmp_path)) + contract.save_memory("my usual coffee order is a flat white.", subject="preferences") + contract.save_memory("the deploy day moved to tuesday.", subject="ops-runbook") + assert contract.search_subjects("prefer") == ["preferences"] + assert contract.search_subjects("runbook") == ["ops-runbook"] + assert contract.search_subjects("r") == ["ops-runbook", "preferences"] + + +def test_search_in_subject_scopes_hits(tmp_path: Path) -> None: + contract = MemoryContract(_kb(tmp_path)) + contract.save_memory( + "the api rate limit is 640 requests per minute.", subject="ops-runbook" + ) + contract.save_memory( + "personal note: my api rate limit worry is overblown.", subject="journal" + ) + hits = contract.search_in_subject("ops-runbook", "api rate limit") + assert hits, "subject-scoped search found nothing" + assert all(h.subject == "ops-runbook" for h in hits) + assert any("640" in h.text for h in hits) From 7a62b73a2ccf3e81573ebe8add1f73cf329b06a0 Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:24:49 +0900 Subject: [PATCH 2/3] docs(competition): refresh ladder baseline for the v2 bench contract the verifiability categories change the category set, so seeds 1-6 rescore: composite 0.52 +- 0.03 replaces 0.57 +- 0.04 as row 0. the koth fold compares only max-bench-version scores, so the reigning-kit number and the challenger number stay on the same contract. --- competition/LEADERBOARD.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/competition/LEADERBOARD.md b/competition/LEADERBOARD.md index cc0204e0..f6e40405 100644 --- a/competition/LEADERBOARD.md +++ b/competition/LEADERBOARD.md @@ -10,7 +10,7 @@ payout rank is settled by the monthly sealed commit-reveal run. | # | champion | PR | dethroned on | scored mean | margin over prior | |---|----------|----|--------------|-------------|-------------------| -| 0 | baseline kit (repo defaults) | — | 2026-07-27 | 0.57 ± 0.04 (seeds 1–6) | — | +| 0 | baseline kit (repo defaults) | — | 2026-07-28 | 0.52 ± 0.03 (seeds 1–6) | — | payouts follow the season shares in docs/vouchbench-seasons.md (65/14/10/7/4). days-on-throne accrue between dethrones; the monthly From 8bcec71eb1294a6888ea72b3deba76a34ba32a3a Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:45:41 +0900 Subject: [PATCH 3/3] feat(competition): engine lane - submit ranking code, scored in a sandbox the engine-submission half of the koth ladder, and the real ditto equivalent: contributors submit a retrieval STRATEGY as python code (a rank(query, candidates, *, limit) -> ids function under contrib/strategies/), and vouchbench scores it. the kit lane only tuned coefficients; this is where a new fusion or reranker actually competes. the load-bearing rule: a kit is data and auto-merges; a strategy is code and NEVER auto-merges. vouch is a library people install, so a winning strategy earns the leaderboard and payout and ships only after human review merges it as a default - the review gate applied to engine code. scoring runs untrusted code, so vouch.strategy.run_sandboxed executes each submission in a python -I child that sets rlimits and installs an audit hook blocking network, subprocess, os.exec/fork, ctypes.dlopen, and filesystem writes before importing the file; the parent enforces a wall-clock timeout and treats any failure as no-reordering. the engine hook in build_context_pack is off by default and byte-identical when unset (retrieval.strategy, a dotted path, wires a merged strategy). koth-engine-gate.yml runs pull_request_target with contents:read, no secrets, no write token, no merge. hardened against an adversarial review. two confirmed defects fixed with regression tests: the audit hook read its blocklists from module globals (disarmable in two lines by reassigning __main__ globals) - now bound in closure cells; and the result travelled on the strategy's own stdout (any print corrupted it, silently scoring a good strategy as a no-op) - the child now writes the result to a private dup of stdout with fd 1 pointed at /dev/null. the docstring is honest that in-process python isolation is best-effort and the real boundary is the ephemeral runner plus human merge. --- .github/scripts/score_strategy.py | 98 +++++++ .github/workflows/koth-engine-gate.yml | 152 +++++++++++ contrib/strategies/README.md | 58 ++++ contrib/strategies/baseline.py | 19 ++ contrib/strategies/example_lexical.py | 34 +++ docs/koth-strategy-lane.md | 96 +++++++ src/vouch/context.py | 70 +++++ src/vouch/strategy.py | 356 +++++++++++++++++++++++++ tests/test_strategy.py | 202 ++++++++++++++ 9 files changed, 1085 insertions(+) create mode 100644 .github/scripts/score_strategy.py create mode 100644 .github/workflows/koth-engine-gate.yml create mode 100644 contrib/strategies/README.md create mode 100644 contrib/strategies/baseline.py create mode 100644 contrib/strategies/example_lexical.py create mode 100644 docs/koth-strategy-lane.md create mode 100644 src/vouch/strategy.py create mode 100644 tests/test_strategy.py diff --git a/.github/scripts/score_strategy.py b/.github/scripts/score_strategy.py new file mode 100644 index 00000000..b3d16d5e --- /dev/null +++ b/.github/scripts/score_strategy.py @@ -0,0 +1,98 @@ +"""Paired champion-vs-challenger scoring for the engine (strategy) lane. + +Same dethrone test as the kit lane (docs/vouchbench-seasons.md), but the arm +is a pluggable ranking strategy instead of a config fragment: each submission +is loaded as an untrusted file and run through vouch.strategy.SandboxProxy, so +the scored code executes only inside the sandbox child. + + dethroned iff mean(challenger - champion) >= max(0.007, 1.96 x SE) + +Exit code 0 = dethroned, 3 = held, 1 = error. Unlike the kit lane, a winning +verdict here does NOT auto-merge: engine code ships only through human review. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import statistics +from pathlib import Path + +from vouch import bench +from vouch.strategy import SandboxProxy + +N_SEEDS = 8 +FLOOR = 0.007 +Z = 1.96 +SESSION_GAP_SECONDS = 2.0 + + +def day_seeds(base_sha: str, date: str, n: int = N_SEEDS) -> list[int]: + seeds = [] + for i in range(n): + digest = hashlib.sha256(f"{base_sha}:{date}:{i}".encode()).hexdigest() + seeds.append(int(digest[:12], 16)) + return seeds + + +def score(path: str, seeds: list[int]) -> list[float]: + proxy = SandboxProxy(path) + return [ + bench.run(s, strategy=proxy, session_gap_seconds=SESSION_GAP_SECONDS)[ + "composite" + ] + for s in seeds + ] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--champion", required=True) + parser.add_argument("--challenger", required=True) + parser.add_argument("--base-sha", required=True) + parser.add_argument("--date", default=None) + parser.add_argument("--out", default=None) + args = parser.parse_args() + + date = args.date or dt.datetime.now(dt.UTC).strftime("%Y-%m-%d") + seeds = day_seeds(args.base_sha, date) + + champion_scores = score(args.champion, seeds) + challenger_scores = score(args.challenger, seeds) + diffs = [ + c - r for c, r in zip(challenger_scores, champion_scores, strict=True) + ] + mean_diff = statistics.mean(diffs) + se = statistics.stdev(diffs) / (len(diffs) ** 0.5) if len(diffs) > 1 else 0.0 + band = max(FLOOR, Z * se) + dethroned = mean_diff >= band + + report = { + "date": date, + "base_sha": args.base_sha, + "seeds": seeds, + "lane": "engine", + "champion": { + "scores": champion_scores, + "mean": statistics.mean(champion_scores), + }, + "challenger": { + "scores": challenger_scores, + "mean": statistics.mean(challenger_scores), + }, + "mean_diff": mean_diff, + "se": se, + "band": band, + "dethroned": dethroned, + } + text = json.dumps(report, indent=1) + print(text) + if args.out: + Path(args.out).write_text(text + "\n", encoding="utf-8") + return 0 if dethroned else 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/koth-engine-gate.yml b/.github/workflows/koth-engine-gate.yml new file mode 100644 index 00000000..6eb8a56f --- /dev/null +++ b/.github/workflows/koth-engine-gate.yml @@ -0,0 +1,152 @@ +# koth engine gate - scores strategy (ranking-code) submissions. +# +# the deliberate contrast with koth-gate.yml (the kit lane): a kit is data +# and auto-merges; a STRATEGY is code and NEVER auto-merges. this job has no +# write token, holds no secrets, and merges nothing. it scores the submission +# in a sandbox and posts a scorecard + leaderboard note; a human reviews the +# code and merges it as a new default if it wins. that human review is the +# review gate applied to engine code. +# +# security model: +# - pull_request_target: the workflow, the grader (score_strategy.py), and +# the champion strategy all come from the BASE branch. only the challenger +# .py is read from the PR, and it is executed ONLY inside the sandbox +# child (vouch.strategy.run_sandboxed: rlimits + an audit hook blocking +# network/subprocess/writes). +# - permissions: contents: read only. even a full sandbox escape lands on +# an ephemeral runner with a read-only token and no secrets, and cannot +# merge itself. +name: koth-engine-gate + +on: + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review] + +concurrency: + group: koth-engine-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + gate: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + pull-requests: write # scorecard comment only - no merge + steps: + - name: checkout base branch (trusted code only) + uses: actions/checkout@v4 + + - name: classify the PR + id: classify + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ + --paginate --jq '.[].filename' > /tmp/changed.txt + count=$(wc -l < /tmp/changed.txt) + only=$(head -1 /tmp/changed.txt) + case "$only" in + contrib/strategies/baseline.py|contrib/strategies/README.md) only="" ;; + esac + if [ "$count" = "1" ] && \ + printf '%s' "$only" | grep -qE '^contrib/strategies/[A-Za-z0-9_]+\.py$'; then + echo "mode=engine" >> "$GITHUB_OUTPUT" + echo "path=$only" >> "$GITHUB_OUTPUT" + else + echo "mode=normal" >> "$GITHUB_OUTPUT" + fi + + - name: pass through (not a strategy PR) + if: steps.classify.outputs.mode == 'normal' + run: echo "not a single-strategy PR - engine gate does not apply." + + - name: fetch challenger strategy from the PR (as data) + if: steps.classify.outputs.mode == 'engine' + env: + GH_TOKEN: ${{ github.token }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + KIT_PATH: ${{ steps.classify.outputs.path }} + run: | + gh api "repos/${HEAD_REPO}/contents/${KIT_PATH}?ref=${HEAD_SHA}" \ + > /tmp/strat-meta.json + encoding=$(jq -r '.encoding // ""' /tmp/strat-meta.json) + if [ "$encoding" != "base64" ]; then + echo "strategy not returned as an inlined blob (encoding=$encoding)" >&2 + exit 1 + fi + jq -r '.content' /tmp/strat-meta.json | base64 -d > /tmp/challenger.py + if [ ! -s /tmp/challenger.py ]; then + echo "fetched strategy is empty" >&2 + exit 1 + fi + + - name: set up python + if: steps.classify.outputs.mode == 'engine' + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: install vouch (base branch code) + if: steps.classify.outputs.mode == 'engine' + run: python -m pip install -e . + + - name: paired scoring - challenger vs baseline champion (sandboxed) + if: steps.classify.outputs.mode == 'engine' + id: score + env: + BASE_SHA: ${{ github.sha }} + run: | + set +e + python .github/scripts/score_strategy.py \ + --champion contrib/strategies/baseline.py \ + --challenger /tmp/challenger.py \ + --base-sha "$BASE_SHA" \ + --out /tmp/engine-report.json + code=$? + set -e + if [ "$code" = "0" ]; then + echo "verdict=dethroned" >> "$GITHUB_OUTPUT" + elif [ "$code" = "3" ]; then + echo "verdict=held" >> "$GITHUB_OUTPUT" + else + exit "$code" + fi + + - name: post the scorecard + if: steps.classify.outputs.mode == 'engine' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + VERDICT: ${{ steps.score.outputs.verdict }} + run: | + { + echo "koth engine lane - ${VERDICT}" + echo + echo '```json' + cat /tmp/engine-report.json + echo '```' + echo + echo "engine code is NOT auto-merged. a winning strategy earns the" + echo "leaderboard place; a maintainer reviews the code and merges it" + echo "as a new default. the daily result is provisional (public seeds)" + echo "- payout rank is settled by the monthly sealed run." + } > /tmp/comment.md + gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --body-file /tmp/comment.md + + - name: report the verdict as the check result + if: steps.classify.outputs.mode == 'engine' + env: + VERDICT: ${{ steps.score.outputs.verdict }} + run: | + echo "verdict: ${VERDICT}" + # a held challenger is a green, informational result - nothing merges + # here regardless, so the gate never blocks. a scoring error already + # failed the job above. + exit 0 diff --git a/contrib/strategies/README.md b/contrib/strategies/README.md new file mode 100644 index 00000000..fa4abbb8 --- /dev/null +++ b/contrib/strategies/README.md @@ -0,0 +1,58 @@ +# contrib/strategies - the engine-submission lane + +this is the ditto-equivalent lane: you submit **real ranking code**, not a +config file. a strategy decides the order the reader sees retrieved +candidates in - the place where a new fusion, a learned reranker, or a novel +signal actually lives. + +## the contract + +your file exposes a `rank` function (or a `STRATEGY` object with a `.rank` +method): + +```python +from vouch.strategy import Candidate + +def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: + # return candidate ids, best first + ... +``` + +- a `Candidate` has `kind`, `id`, `summary`, `score` - **data only**. your + code never gets the KB, the filesystem, or the network. +- ordering is authoritative but bounded: ids you invent are ignored, and any + candidate you omit is appended in its original order. you can reorder and + de-prioritise; you cannot fabricate or hide a result. +- [`baseline.py`](./baseline.py) is the reigning champion (returns the + backend order unchanged). [`example_lexical.py`](./example_lexical.py) is a + worked example you can study and beat. + +## how it is scored + +open a PR that touches **only** your new file under `contrib/strategies/`. +the `koth-engine-gate` workflow: + +1. runs your code in a locked-down `python -I` sandbox (resource limits + an + audit hook that blocks network, subprocess, and filesystem writes); +2. scores vouchbench with your strategy vs the reigning champion, paired over + the day's seeds; +3. posts the scorecard and updates the engine leaderboard on a win. + +## the one hard rule: engine code is never auto-merged + +the config (kit) lane auto-merges because a kit cannot execute. **strategy +code can**, and vouch is a library people install - so a winning strategy is +never merged automatically. it earns the leaderboard place and the payout, +and ships only after a human reviews the code and merges it as a new default. +that human review is vouch's version of ditto's tee-plus-deployment gate: the +benchmark decides the *rank*, a person decides what *ships*. + +reproduce any score locally: + +```bash +pip install -e . +python .github/scripts/score_strategy.py \ + --champion contrib/strategies/baseline.py \ + --challenger contrib/strategies/example_lexical.py \ + --base-sha "$(git rev-parse origin/main)" --date "$(date -u +%F)" +``` diff --git a/contrib/strategies/baseline.py b/contrib/strategies/baseline.py new file mode 100644 index 00000000..e74d7d09 --- /dev/null +++ b/contrib/strategies/baseline.py @@ -0,0 +1,19 @@ +"""The reigning engine-lane champion: trust the backend's fused order. + +This is the strategy every submission must beat. It returns the candidates in +exactly the order retrieval handed them over - i.e. it does nothing - so a +challenger only dethrones it by making vouch's benchmark score genuinely +higher, not by accident. + +A strategy is real ranking code. It receives the query and the retrieved +candidates (data only - no KB, no disk, no network) and returns the ids in the +order the reader should see them. Ordering is authoritative but bounded: ids +you invent are ignored, and any candidate you drop is appended at the tail, so +you can reorder and de-prioritise but never fabricate or hide a result. +""" + +from vouch.strategy import Candidate + + +def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: + return [c.id for c in candidates] diff --git a/contrib/strategies/example_lexical.py b/contrib/strategies/example_lexical.py new file mode 100644 index 00000000..aead101c --- /dev/null +++ b/contrib/strategies/example_lexical.py @@ -0,0 +1,34 @@ +"""A worked example: re-rank by lexical overlap with the query. + +Not necessarily a winner - it exists to show the shape of a real submission. +It boosts candidates whose summary shares more words with the query, blended +with the backend's own score so a strong retrieval signal is not thrown away. + +Copy this file, change the scoring, and open a PR that touches only your new +file under contrib/strategies/. The engine gate scores it in a sandbox against +the reigning champion; you never edit the engine itself. +""" + +import re + +from vouch.strategy import Candidate + +_WORD = re.compile(r"[a-z0-9]+") + + +def _tokens(text: str) -> set[str]: + return set(_WORD.findall(text.lower())) + + +def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: + q = _tokens(query) + if not q: + return [c.id for c in candidates] + + def blended(c: Candidate) -> float: + overlap = len(q & _tokens(c.summary)) / len(q) + # keep the backend score in the mix; overlap only tips ties and + # rescues a lexically-obvious hit the fusion under-ranked. + return 0.7 * c.score + 0.3 * overlap + + return [c.id for c in sorted(candidates, key=blended, reverse=True)] diff --git a/docs/koth-strategy-lane.md b/docs/koth-strategy-lane.md new file mode 100644 index 00000000..3adf35b4 --- /dev/null +++ b/docs/koth-strategy-lane.md @@ -0,0 +1,96 @@ +# the engine lane - submit ranking code, not config + +the kit ladder (docs/koth-ladder.md) lets contributors tune coefficients. +this lane is the ditto-equivalent: contributors submit **real engine code** - +a retrieval strategy that decides the order the reader sees candidates in - +and the benchmark scores it. it is the place a new fusion, a learned reranker, +or a novel signal actually competes. + +## why a second lane exists + +a kit can only move dials that already exist, so its ceiling is low - most +single knobs are no-ops on the current bench. real gains come from new ranking +logic. ditto accepts exactly this (miners submit the retrieval harness). the +difference is that ditto is a hosted service scored in a tee, while vouch is a +library people install - so the two lanes draw the trust boundary in different +places: + +| | kit lane | engine lane | +|---|---|---| +| what you submit | `competition/kits/current/kit.yaml` (data) | `contrib/strategies/.py` (code) | +| scored by | vouchbench, config arm | vouchbench, sandboxed strategy arm | +| on a win | **auto-merges** (data cannot execute) | leaderboard + payout; **never auto-merges** | +| how it ships | promoted to defaults by a human PR | reviewed and merged by a human | + +## the interface + +```python +from vouch.strategy import Candidate + +def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]: + # return candidate ids, best first + ... +``` + +a `Candidate` is `kind`, `id`, `summary`, `score` - **data only**. the +strategy never receives the KB, the filesystem, or a socket. ordering is +authoritative but bounded (ids you invent are ignored; candidates you omit are +appended), so a strategy can reorder and de-prioritise but cannot fabricate or +hide a result. see `contrib/strategies/baseline.py` (the champion) and +`example_lexical.py` (a worked example). + +## how a submission is scored + +open a PR touching only your new `contrib/strategies/.py`. the +`koth-engine-gate` workflow runs your code through +`vouch.strategy.run_sandboxed` and scores vouchbench with it, paired against +the reigning champion over the day's seeds, then posts the scorecard. + +### the sandbox + +each `rank` call runs in a fresh `python -I` child that, before importing your +file, installs: + +- **resource limits** (`RLIMIT_CPU`, `RLIMIT_AS`, `RLIMIT_NOFILE`) and a + parent-side wall-clock timeout - a runaway or memory-hungry strategy is + killed, not the run; +- **an audit hook** (`sys.addaudithook`) that refuses network + (`socket.*`/`urllib.*`), process spawning (`subprocess`, `os.exec`/`fork`, + native `ctypes.dlopen`), and filesystem writes (any `open` with write + intent). reads are allowed so numpy and friends still import. + +a crash, a timeout, or a blocked call yields "no reordering" - a broken +strategy simply fails to improve; it cannot take down scoring. + +### the honest boundary + +an in-process python guard cannot stop a determined native-code escape. it is +defence in depth, not the trust root. the real boundary is the same one ditto +relies on: the scoring job runs on an ephemeral CI runner with a **read-only +token and no secrets**, and **engine code is never auto-merged**. the benchmark +decides the *rank*; a human reviewing the diff decides what *ships*. if you +ever score submissions off ephemeral CI, add OS-level isolation (a container, +nsjail, or gVisor) before trusting the audit hook alone. + +## reproduce locally + +```bash +pip install -e . +python .github/scripts/score_strategy.py \ + --champion contrib/strategies/baseline.py \ + --challenger contrib/strategies/example_lexical.py \ + --base-sha "$(git rev-parse origin/main)" --date "$(date -u +%F)" +``` + +## shipping a winner (maintainer) + +1. read the strategy code - it is about to run on every user's machine. +2. move it into `src/vouch/strategies/` and point `retrieval.strategy` (a + dotted import path) at it in the starter config, or wire it as the default. + the in-engine hook loads a shipped strategy in-process (no sandbox - it is + now trusted, reviewed code). +3. record the dethrone in the engine ladder's `LEADERBOARD`. + +the config knob `retrieval.strategy` is how a merged strategy actually changes +retrieval; until one is merged it is unset and retrieval is byte-identical to +today. diff --git a/src/vouch/context.py b/src/vouch/context.py index 089701ac..a715ca7e 100644 --- a/src/vouch/context.py +++ b/src/vouch/context.py @@ -20,6 +20,7 @@ import yaml from . import graph, hot_memory, index_db, retrieval_events +from . import strategy as strategy_mod from .embeddings.fusion import rrf_fuse from .models import ClaimStatus, ContextItem, ContextPack, ContextQuality from .scoping import ( @@ -308,6 +309,71 @@ def _maybe_rerank( return ordered + hits[window_size:] +def _configured_strategy(store: KBStore) -> str | None: + """Resolve ``retrieval.strategy`` - a dotted import path to a shipped, + human-merged strategy - from config.yaml. Off (None) by default.""" + try: + loaded = yaml.safe_load(store.config_path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError): + return None + if not isinstance(loaded, dict): + return None + retrieval = loaded.get("retrieval") + if not isinstance(retrieval, dict): + return None + dotted = retrieval.get("strategy") + return dotted if isinstance(dotted, str) and dotted else None + + +def _maybe_strategy( + store: KBStore, + *, + query: str, + hits: list[tuple[str, str, str, float, str]], + limit: int, + strategy: strategy_mod.RetrievalStrategy | None = None, +) -> list[tuple[str, str, str, float, str]]: + """Apply a pluggable ranking strategy as the final reorder stage. + + ``strategy`` is passed explicitly by the benchmark (an untrusted + submission, wrapped in a sandbox proxy); otherwise a shipped strategy is + resolved from ``retrieval.strategy`` config and loaded in-process. With + neither, hits pass through byte-identical. A strategy that raises or + returns nothing usable leaves the order untouched - retrieval never fails + because a ranking plugin misbehaved. + """ + strat = strategy + if strat is None: + dotted = _configured_strategy(store) + if not dotted: + return hits + try: + strat = strategy_mod.load_dotted(dotted) + except Exception: + return hits + if not hits: + return hits + # the strategy addresses hits by id; if two hits somehow share one (a + # cross-kind slug collision), a reorder-by-id would be ambiguous, so skip + # the stage rather than risk attaching an order to the wrong artifact. + ids = [h[1] for h in hits] + if len(set(ids)) != len(ids): + return hits + candidates = [ + strategy_mod.Candidate(kind=k, id=i, summary=s, score=sc) + for k, i, s, sc, _b in hits + ] + try: + ordered_ids = strat.rank(query, candidates, limit=limit) + except Exception: + return hits + by_id = {h[1]: h for h in hits} + reordered = strategy_mod.apply_ordering( + list(ordered_ids), [(k, i, s, sc) for k, i, s, sc, _b in hits] + ) + return [by_id[h4[1]] for h4 in reordered] + + def _retrieve( store: KBStore, query: str, @@ -602,6 +668,7 @@ def build_context_pack( graph_depth: int = 1, graph_limit: int = 20, graph_rel_types: list[str] | None = None, + strategy: strategy_mod.RetrievalStrategy | None = None, ) -> ContextPack | dict[str, Any]: viewer = viewer_from( config_path=store.config_path, @@ -609,6 +676,9 @@ def build_context_pack( agent=agent, ) hits = _retrieve(store, query, limit, viewer) + hits = _maybe_strategy( + store, query=query, hits=hits, limit=limit, strategy=strategy + ) items: list[ContextItem] = [] for kind, hid, summary, score, backend in hits: cites: list[str] = [] diff --git a/src/vouch/strategy.py b/src/vouch/strategy.py new file mode 100644 index 00000000..35d0a090 --- /dev/null +++ b/src/vouch/strategy.py @@ -0,0 +1,356 @@ +"""Pluggable retrieval strategies - the engine-submission lane of the koth +ladder. + +A *strategy* is real ranking code: given the query and the candidate hits a +backend retrieved, it returns the ids in the order the reader should see them. +This is where a contributor competes on algorithm (a new fusion, a learned +reranker, a novel signal) rather than on config coefficients (the kit lane). + +Two ways a strategy runs, with very different trust: + +- **shipped / merged** (trusted): resolved from ``retrieval.strategy`` in + config.yaml as a dotted import path, loaded in-process. It is trusted + because a human reviewed and merged it - the review gate, applied to code. +- **a competition submission** (untrusted): a file under ``contrib/strategies/`` + loaded by the benchmark and run via :func:`run_sandboxed`, which executes it + in a separate ``python -I`` process under resource limits and an audit hook + that blocks network, subprocess, and filesystem writes. + +Honesty about the sandbox boundary: the audit hook + rlimits stop casual and +accidental misbehaviour and raise the bar against a hostile one, but no +in-process Python guard is a true boundary - a determined attacker who can +introspect the interpreter (native ctypes/cffi, or pure-Python gc/frame walking +to reach and neutralise the hook object) can defeat it. The *real* boundary is +the same one ditto leans on: the scoring job runs on an ephemeral CI runner +with a read-only token and no secrets, and **strategy code is never +auto-merged** - it earns a leaderboard place and is shipped only through human +review. So the worst a full escape achieves during scoring is misbehaviour on a +throwaway runner that can reach nothing and merge nothing. The sandbox is +defence in depth; the trust root is the runner's least privilege plus the human +merge gate. +""" + +from __future__ import annotations + +import importlib +import importlib.util +import json +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol, runtime_checkable + +# a retrieval hit as the reorder stages pass it: (kind, id, summary, score). +Hit = tuple[str, str, str, float] + +DEFAULT_TIMEOUT_S = 20.0 +DEFAULT_MEM_MB = 2048 +DEFAULT_CPU_S = 15 + +# audit events refused inside the sandbox child. network, process spawning, +# and native dlopen are blocked outright; filesystem writes are blocked by +# inspecting the open mode (reads - needed to import numpy etc. - are allowed). +_BLOCKED_EXACT = frozenset({ + "os.system", + "os.exec", + "os.fork", + "os.forkpty", + "os.posix_spawn", + "os.spawn", + "subprocess.Popen", + "ctypes.dlopen", + "ctypes.dlsym", + "ctypes.call_function", + "ctypes.cdata", +}) +_BLOCKED_PREFIXES = ("socket.", "urllib.", "ftplib.", "http.") + + +@dataclass(frozen=True) +class Candidate: + """What a strategy sees for one retrieved artifact - data only. + + A strategy never receives the KB, the filesystem, or a network handle; + just these fields, so it cannot reach anything outside the ranking task. + """ + + kind: str + id: str + summary: str + score: float + + +@runtime_checkable +class RetrievalStrategy(Protocol): + """The interface a submission implements. + + ``rank`` returns candidate ids best-first. It is treated as *ordering + only*: ids not in the input are ignored, and any input id the strategy + omits is appended in its original order, so a strategy can reorder and + drop-from-the-top but can never fabricate a result or smuggle one in. + """ + + def rank( + self, query: str, candidates: list[Candidate], *, limit: int + ) -> list[str]: ... + + +def apply_ordering(ordered_ids: list[str], hits: list[Hit]) -> list[Hit]: + """Reorder ``hits`` by ``ordered_ids``, preserving the hit set. + + Mirrors the discipline of the built-in rerank stage: the strategy may move + artifacts but not add or remove them. Unknown ids are dropped; hits the + strategy did not mention keep their relative order at the tail. + """ + by_id: dict[str, Hit] = {} + for hit in hits: + by_id.setdefault(hit[1], hit) + seen: set[str] = set() + ordered: list[Hit] = [] + for hid in ordered_ids: + match = by_id.get(hid) + if match is not None and hid not in seen: + ordered.append(match) + seen.add(hid) + for hit in hits: + if hit[1] not in seen: + ordered.append(hit) + seen.add(hit[1]) + return ordered + + +def load_from_path(path: str | Path) -> RetrievalStrategy: + """Import a strategy from a .py file and return its strategy object. + + The module must expose either a ``STRATEGY`` object with a ``rank`` method + or a module-level ``rank`` function. Loading runs module top-level code, so + only call this on trusted (merged) files or inside the sandbox child. + """ + path = Path(path) + spec = importlib.util.spec_from_file_location(f"vouch_strategy_{path.stem}", path) + if spec is None or spec.loader is None: + raise ValueError(f"cannot load strategy from {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return _strategy_from_module(module) + + +def load_dotted(dotted: str) -> RetrievalStrategy: + """Import a trusted, shipped strategy by dotted module path (config hook).""" + module = importlib.import_module(dotted) + return _strategy_from_module(module) + + +def _strategy_from_module(module: Any) -> RetrievalStrategy: + candidate = getattr(module, "STRATEGY", None) + if candidate is not None and hasattr(candidate, "rank"): + return candidate # type: ignore[no-any-return] + fn = getattr(module, "rank", None) + if callable(fn): + rank_fn = fn + + class _FnStrategy: + def rank( + self, query: str, candidates: list[Candidate], *, limit: int + ) -> list[str]: + return list(rank_fn(query, candidates, limit=limit)) + + return _FnStrategy() + raise ValueError( + "strategy module must define STRATEGY (with .rank) or a rank() function" + ) + + +class SandboxProxy: + """A strategy handle whose ``rank`` runs the real code in a sandbox child. + + Built by the benchmark for an untrusted submission. Each ``rank`` call + spawns a fresh ``python -I`` process, so a submission cannot keep state + between calls or wear down the limits over time. A crash, a timeout, or a + limit breach yields an empty ordering, which :func:`apply_ordering` turns + into "keep the backend's order" - a broken strategy simply fails to + improve rather than taking down the run. + """ + + def __init__( + self, + path: str | Path, + *, + timeout_s: float = DEFAULT_TIMEOUT_S, + mem_mb: int = DEFAULT_MEM_MB, + cpu_s: int = DEFAULT_CPU_S, + ) -> None: + self.path = str(Path(path).resolve()) + self.timeout_s = timeout_s + self.mem_mb = mem_mb + self.cpu_s = cpu_s + self.failures = 0 + + def rank( + self, query: str, candidates: list[Candidate], *, limit: int + ) -> list[str]: + ordered = run_sandboxed( + self.path, + query, + candidates, + limit=limit, + timeout_s=self.timeout_s, + mem_mb=self.mem_mb, + cpu_s=self.cpu_s, + ) + if ordered is None: + self.failures += 1 + return [] + return ordered + + +def run_sandboxed( + path: str, + query: str, + candidates: list[Candidate], + *, + limit: int, + timeout_s: float = DEFAULT_TIMEOUT_S, + mem_mb: int = DEFAULT_MEM_MB, + cpu_s: int = DEFAULT_CPU_S, +) -> list[str] | None: + """Run a strategy file in a locked-down child; return ordered ids or None. + + None means the child failed (crash, timeout, limit, or malformed output) - + the caller treats that as "no reordering", never as an error that aborts + scoring. + """ + payload = { + "path": path, + "query": query, + "limit": limit, + "mem_bytes": mem_mb * 1024 * 1024, + "cpu_seconds": cpu_s, + "candidates": [ + {"kind": c.kind, "id": c.id, "summary": c.summary, "score": c.score} + for c in candidates + ], + } + try: + proc = subprocess.run( + [sys.executable, "-I", "-m", "vouch.strategy", "--child"], + input=json.dumps(payload), + capture_output=True, + text=True, + timeout=timeout_s, + # a minimal env; -I already ignores PYTHON* vars and the cwd. + env={"PATH": "/usr/bin:/bin"}, + ) + except (subprocess.TimeoutExpired, OSError): + return None + if proc.returncode != 0: + return None + try: + result = json.loads(proc.stdout) + ordered = result["ordered"] + except (json.JSONDecodeError, KeyError, TypeError): + return None + if not isinstance(ordered, list): + return None + return [str(x) for x in ordered] + + +# --- sandbox child --------------------------------------------------------- + + +def _install_audit_hook() -> None: + import os + + # Bind the guard sets into closure cells that hold the original frozenset + # objects. The hook reads them via LOAD_DEREF, not from module globals, so + # untrusted top-level code cannot disarm the block by reassigning + # ``__main__._BLOCKED_EXACT`` (the child runs as __main__). This is still + # in-process defence in depth - see the module docstring: a determined + # attacker who introspects the interpreter can defeat any pure-Python + # hook, which is why the real boundary is the ephemeral runner and the + # no-auto-merge rule, not this function. + blocked_exact = _BLOCKED_EXACT + blocked_prefixes = _BLOCKED_PREFIXES + + def _hook(event: str, args: tuple[Any, ...]) -> None: + if event in blocked_exact or event.startswith(blocked_prefixes): + raise PermissionError(f"blocked in strategy sandbox: {event}") + if event == "open": + # args = (path, mode, flags); refuse any write/append/create intent. + mode = args[1] if len(args) > 1 else "" + if isinstance(mode, str) and any(c in mode for c in "wax+"): + raise PermissionError("filesystem writes are blocked in the sandbox") + flags = args[2] if len(args) > 2 else 0 + if isinstance(flags, int) and ( + flags & (os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_APPEND) + ): + raise PermissionError("filesystem writes are blocked in the sandbox") + + sys.addaudithook(_hook) + + +def _apply_rlimits(mem_bytes: int, cpu_seconds: int) -> None: + try: + import resource + except ImportError: # non-unix; the audit hook still applies + return + # cpu seconds is a hard ceiling backing the parent's wall-clock timeout; + # nofile caps fd pressure. RLIMIT_AS is deliberately generous - numpy and + # friends reserve large virtual space even at small RSS, and a too-tight + # AS limit kills honest strategies, so wall-clock + cpu are the real caps. + for res, soft in ( + (resource.RLIMIT_CPU, cpu_seconds), + (resource.RLIMIT_AS, mem_bytes), + (resource.RLIMIT_NOFILE, 64), + ): + try: + hard = resource.getrlimit(res)[1] + cap = soft if hard == resource.RLIM_INFINITY else min(soft, hard) + resource.setrlimit(res, (cap, hard)) + except (ValueError, OSError): + pass + + +def _child_main() -> int: + import os + + payload = json.load(sys.stdin) + mem_bytes = int(payload["mem_bytes"]) + cpu_seconds = int(payload["cpu_seconds"]) + path = payload["path"] + query = payload["query"] + limit = int(payload["limit"]) + candidates = [ + Candidate(kind=c["kind"], id=c["id"], summary=c["summary"], score=c["score"]) + for c in payload["candidates"] + ] + # Isolate the result channel from the strategy's stdout: a strategy that + # prints (a debug line, a library banner, a progress bar) must not corrupt + # the JSON the parent parses. Keep a private dup of the real stdout for the + # result, then point fd 1 at /dev/null so anything the strategy writes is + # discarded. Both opens happen before the write-blocking audit hook arms. + result_fd = os.dup(1) + devnull_fd = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull_fd, 1) + # limits and the audit hook are armed BEFORE the untrusted module is loaded. + _apply_rlimits(mem_bytes, cpu_seconds) + _install_audit_hook() + strategy = load_from_path(path) + ordered = strategy.rank(query, candidates, limit=limit) + valid = {c.id for c in candidates} + ordered_ids = [str(x) for x in ordered if str(x) in valid] + os.write(result_fd, json.dumps({"ordered": ordered_ids}).encode("utf-8")) + return 0 + + +def main(argv: list[str] | None = None) -> int: + argv = sys.argv[1:] if argv is None else argv + if argv and argv[0] == "--child": + return _child_main() + print("usage: python -I -m vouch.strategy --child (reads a job on stdin)") + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_strategy.py b/tests/test_strategy.py new file mode 100644 index 00000000..103ffecf --- /dev/null +++ b/tests/test_strategy.py @@ -0,0 +1,202 @@ +"""The pluggable-strategy engine lane, with the sandbox as the load-bearing +part. Every escape test here guards the boundary that lets vouch score +untrusted ranking code automatically.""" + +from pathlib import Path + +import pytest + +from vouch.strategy import ( + Candidate, + SandboxProxy, + apply_ordering, + load_from_path, + run_sandboxed, +) + +REPO = Path(__file__).resolve().parents[1] +BASELINE = str(REPO / "contrib" / "strategies" / "baseline.py") +EXAMPLE = str(REPO / "contrib" / "strategies" / "example_lexical.py") + +CANDS = [ + Candidate("claim", "a", "alpha beta", 0.9), + Candidate("claim", "b", "gamma delta", 0.5), + Candidate("page", "c", "beta gamma query", 0.3), +] + + +def _write(tmp_path: Path, body: str) -> str: + p = tmp_path / "challenger.py" + p.write_text(body, encoding="utf-8") + return str(p) + + +# --- interface + ordering discipline -------------------------------------- + + +def test_baseline_is_identity() -> None: + strat = load_from_path(BASELINE) + assert strat.rank("beta", CANDS, limit=10) == ["a", "b", "c"] + + +def test_apply_ordering_drops_unknown_and_appends_missing() -> None: + hits = [(c.kind, c.id, c.summary, c.score) for c in CANDS] + out = apply_ordering(["c", "invented", "a"], hits) + # 'invented' dropped, 'b' (unmentioned) appended at the tail. + assert [h[1] for h in out] == ["c", "a", "b"] + + +def test_apply_ordering_cannot_grow_the_set() -> None: + hits = [(c.kind, c.id, c.summary, c.score) for c in CANDS] + out = apply_ordering(["a", "a", "b", "c", "c"], hits) + assert sorted(h[1] for h in out) == ["a", "b", "c"] + + +# --- sandbox happy path ---------------------------------------------------- + + +def test_sandbox_runs_and_matches_in_process() -> None: + assert run_sandboxed(BASELINE, "beta", CANDS, limit=10) == ["a", "b", "c"] + + +def test_sandbox_is_deterministic() -> None: + a = run_sandboxed(EXAMPLE, "beta gamma query", CANDS, limit=10) + b = run_sandboxed(EXAMPLE, "beta gamma query", CANDS, limit=10) + assert a == b and a is not None + + +def test_sandbox_only_returns_known_ids() -> None: + body = """ +from vouch.strategy import Candidate +def rank(query, candidates, *, limit): + return ["a", "ghost", "c"] # 'ghost' is not a candidate +""" + # the child filters to valid ids before returning. + import tempfile + + with tempfile.TemporaryDirectory() as d: + path = _write(Path(d), body) + assert run_sandboxed(path, "q", CANDS, limit=10) == ["a", "c"] + + +# --- sandbox blocks (the security contract) ------------------------------- + + +def test_sandbox_blocks_network(tmp_path: Path) -> None: + body = """ +from vouch.strategy import Candidate +def rank(query, candidates, *, limit): + import socket + socket.socket().connect(("1.1.1.1", 80)) + return [c.id for c in candidates] +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) is None + + +def test_sandbox_blocks_file_write(tmp_path: Path) -> None: + target = tmp_path / "pwned" + body = f""" +from vouch.strategy import Candidate +def rank(query, candidates, *, limit): + open({str(target)!r}, "w").write("x") + return [c.id for c in candidates] +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) is None + assert not target.exists() + + +def test_sandbox_blocks_subprocess(tmp_path: Path) -> None: + body = """ +from vouch.strategy import Candidate +def rank(query, candidates, *, limit): + import subprocess + subprocess.Popen(["/bin/echo", "hi"]) + return [c.id for c in candidates] +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) is None + + +def test_sandbox_kills_infinite_loop(tmp_path: Path) -> None: + body = """ +from vouch.strategy import Candidate +def rank(query, candidates, *, limit): + while True: + pass +""" + assert ( + run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10, timeout_s=5) + is None + ) + + +def test_audit_hook_survives_module_global_disarm(tmp_path: Path) -> None: + # the child runs as __main__; reassigning the guard globals must NOT + # re-enable network/exec, because the hook reads them from closure cells + # holding the original frozensets. + body = """ +import sys +_m = sys.modules["__main__"] +_m._BLOCKED_EXACT = frozenset() +_m._BLOCKED_PREFIXES = () +def rank(query, candidates, *, limit): + import socket + socket.socket().connect(("1.1.1.1", 80)) + return [c.id for c in candidates] +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) is None + + +def test_strategy_stdout_does_not_corrupt_result(tmp_path: Path) -> None: + # a strategy that prints debug output must still have its reordering + # respected - the result travels on a channel the strategy cannot dirty. + body = """ +import sys +def rank(query, candidates, *, limit): + print("noisy debug line") + sys.stdout.write("more noise\\n") + return list(reversed([c.id for c in candidates])) +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) == [ + "c", + "b", + "a", + ] + + +def test_sandbox_survives_a_crashing_strategy(tmp_path: Path) -> None: + body = """ +def rank(query, candidates, *, limit): + raise RuntimeError("boom") +""" + assert run_sandboxed(_write(tmp_path, body), "q", CANDS, limit=10) is None + + +def test_proxy_counts_failures_and_returns_empty(tmp_path: Path) -> None: + body = """ +def rank(query, candidates, *, limit): + raise RuntimeError("boom") +""" + proxy = SandboxProxy(_write(tmp_path, body)) + assert proxy.rank("q", CANDS, limit=10) == [] + assert proxy.failures == 1 + + +# --- retrieval integration ------------------------------------------------- + + +def test_build_context_pack_strategy_none_is_default(tmp_path: Path) -> None: + # strategy=None must not change retrieval - exercised in bench parity, but + # assert the config hook returns None cleanly on a bare KB here. + from vouch.context import _configured_strategy + from vouch.storage import KBStore + + store = KBStore.init(tmp_path / "kb") + assert _configured_strategy(store) is None + + +@pytest.mark.parametrize("path", [BASELINE, EXAMPLE]) +def test_shipped_examples_load(path: str) -> None: + strat = load_from_path(path) + assert hasattr(strat, "rank") + result = strat.rank("beta gamma", CANDS, limit=10) + assert sorted(result) == ["a", "b", "c"]