feat(competition): engine lane - submit ranking code, scored in a sandbox

.github/scripts/score_strategy.py

@@ -0,0 +1,98 @@
+"""Paired champion-vs-challenger scoring for the engine (strategy) lane.
+
+Same dethrone test as the kit lane (docs/vouchbench-seasons.md), but the arm
+is a pluggable ranking strategy instead of a config fragment: each submission
+is loaded as an untrusted file and run through vouch.strategy.SandboxProxy, so
+the scored code executes only inside the sandbox child.
+
+    dethroned  iff  mean(challenger - champion)  >=  max(0.007, 1.96 x SE)
+
+Exit code 0 = dethroned, 3 = held, 1 = error. Unlike the kit lane, a winning
+verdict here does NOT auto-merge: engine code ships only through human review.
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import hashlib
+import json
+import statistics
+from pathlib import Path
+
+from vouch import bench
+from vouch.strategy import SandboxProxy
+
+N_SEEDS = 8
+FLOOR = 0.007
+Z = 1.96
+SESSION_GAP_SECONDS = 2.0
+
+
+def day_seeds(base_sha: str, date: str, n: int = N_SEEDS) -> list[int]:
+    seeds = []
+    for i in range(n):
+        digest = hashlib.sha256(f"{base_sha}:{date}:{i}".encode()).hexdigest()
+        seeds.append(int(digest[:12], 16))
+    return seeds
+
+
+def score(path: str, seeds: list[int]) -> list[float]:
+    proxy = SandboxProxy(path)
+    return [
+        bench.run(s, strategy=proxy, session_gap_seconds=SESSION_GAP_SECONDS)[
+            "composite"
+        ]
+        for s in seeds
+    ]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--champion", required=True)
+    parser.add_argument("--challenger", required=True)
+    parser.add_argument("--base-sha", required=True)
+    parser.add_argument("--date", default=None)
+    parser.add_argument("--out", default=None)
+    args = parser.parse_args()
+
+    date = args.date or dt.datetime.now(dt.UTC).strftime("%Y-%m-%d")
+    seeds = day_seeds(args.base_sha, date)
+
+    champion_scores = score(args.champion, seeds)
+    challenger_scores = score(args.challenger, seeds)
+    diffs = [
+        c - r for c, r in zip(challenger_scores, champion_scores, strict=True)
+    ]
+    mean_diff = statistics.mean(diffs)
+    se = statistics.stdev(diffs) / (len(diffs) ** 0.5) if len(diffs) > 1 else 0.0
+    band = max(FLOOR, Z * se)
+    dethroned = mean_diff >= band
+
+    report = {
+        "date": date,
+        "base_sha": args.base_sha,
+        "seeds": seeds,
+        "lane": "engine",
+        "champion": {
+            "scores": champion_scores,
+            "mean": statistics.mean(champion_scores),
+        },
+        "challenger": {
+            "scores": challenger_scores,
+            "mean": statistics.mean(challenger_scores),
+        },
+        "mean_diff": mean_diff,
+        "se": se,
+        "band": band,
+        "dethroned": dethroned,
+    }
+    text = json.dumps(report, indent=1)
+    print(text)
+    if args.out:
+        Path(args.out).write_text(text + "\n", encoding="utf-8")
+    return 0 if dethroned else 3
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/workflows/koth-engine-gate.yml

@@ -0,0 +1,152 @@
+# koth engine gate - scores strategy (ranking-code) submissions.
+#
+# the deliberate contrast with koth-gate.yml (the kit lane): a kit is data
+# and auto-merges; a STRATEGY is code and NEVER auto-merges. this job has no
+# write token, holds no secrets, and merges nothing. it scores the submission
+# in a sandbox and posts a scorecard + leaderboard note; a human reviews the
+# code and merges it as a new default if it wins. that human review is the
+# review gate applied to engine code.
+#
+# security model:
+#   - pull_request_target: the workflow, the grader (score_strategy.py), and
+#     the champion strategy all come from the BASE branch. only the challenger
+#     .py is read from the PR, and it is executed ONLY inside the sandbox
+#     child (vouch.strategy.run_sandboxed: rlimits + an audit hook blocking
+#     network/subprocess/writes).
+#   - permissions: contents: read only. even a full sandbox escape lands on
+#     an ephemeral runner with a read-only token and no secrets, and cannot
+#     merge itself.
+name: koth-engine-gate
+
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened, ready_for_review]
+
+concurrency:
+  group: koth-engine-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  gate:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      pull-requests: write   # scorecard comment only - no merge
+    steps:
+      - name: checkout base branch (trusted code only)
+        uses: actions/checkout@v4
+
+      - name: classify the PR
+        id: classify
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \
+            --paginate --jq '.[].filename' > /tmp/changed.txt
+          count=$(wc -l < /tmp/changed.txt)
+          only=$(head -1 /tmp/changed.txt)
+          case "$only" in
+            contrib/strategies/baseline.py|contrib/strategies/README.md) only="" ;;
+          esac
+          if [ "$count" = "1" ] && \
+             printf '%s' "$only" | grep -qE '^contrib/strategies/[A-Za-z0-9_]+\.py$'; then
+            echo "mode=engine" >> "$GITHUB_OUTPUT"
+            echo "path=$only" >> "$GITHUB_OUTPUT"
+          else
+            echo "mode=normal" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: pass through (not a strategy PR)
+        if: steps.classify.outputs.mode == 'normal'
+        run: echo "not a single-strategy PR - engine gate does not apply."
+
+      - name: fetch challenger strategy from the PR (as data)
+        if: steps.classify.outputs.mode == 'engine'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          KIT_PATH: ${{ steps.classify.outputs.path }}
+        run: |
+          gh api "repos/${HEAD_REPO}/contents/${KIT_PATH}?ref=${HEAD_SHA}" \
+            > /tmp/strat-meta.json
+          encoding=$(jq -r '.encoding // ""' /tmp/strat-meta.json)
+          if [ "$encoding" != "base64" ]; then
+            echo "strategy not returned as an inlined blob (encoding=$encoding)" >&2
+            exit 1
+          fi
+          jq -r '.content' /tmp/strat-meta.json | base64 -d > /tmp/challenger.py
+          if [ ! -s /tmp/challenger.py ]; then
+            echo "fetched strategy is empty" >&2
+            exit 1
+          fi
+
+      - name: set up python
+        if: steps.classify.outputs.mode == 'engine'
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: install vouch (base branch code)
+        if: steps.classify.outputs.mode == 'engine'
+        run: python -m pip install -e .
+
+      - name: paired scoring - challenger vs baseline champion (sandboxed)
+        if: steps.classify.outputs.mode == 'engine'
+        id: score
+        env:
+          BASE_SHA: ${{ github.sha }}
+        run: |
+          set +e
+          python .github/scripts/score_strategy.py \
+            --champion contrib/strategies/baseline.py \
+            --challenger /tmp/challenger.py \
+            --base-sha "$BASE_SHA" \
+            --out /tmp/engine-report.json
+          code=$?
+          set -e
+          if [ "$code" = "0" ]; then
+            echo "verdict=dethroned" >> "$GITHUB_OUTPUT"
+          elif [ "$code" = "3" ]; then
+            echo "verdict=held" >> "$GITHUB_OUTPUT"
+          else
+            exit "$code"
+          fi
+
+      - name: post the scorecard
+        if: steps.classify.outputs.mode == 'engine'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          VERDICT: ${{ steps.score.outputs.verdict }}
+        run: |
+          {
+            echo "koth engine lane - ${VERDICT}"
+            echo
+            echo '```json'
+            cat /tmp/engine-report.json
+            echo '```'
+            echo
+            echo "engine code is NOT auto-merged. a winning strategy earns the"
+            echo "leaderboard place; a maintainer reviews the code and merges it"
+            echo "as a new default. the daily result is provisional (public seeds)"
+            echo "- payout rank is settled by the monthly sealed run."
+          } > /tmp/comment.md
+          gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \
+            --body-file /tmp/comment.md
+
+      - name: report the verdict as the check result
+        if: steps.classify.outputs.mode == 'engine'
+        env:
+          VERDICT: ${{ steps.score.outputs.verdict }}
+        run: |
+          echo "verdict: ${VERDICT}"
+          # a held challenger is a green, informational result - nothing merges
+          # here regardless, so the gate never blocks. a scoring error already
+          # failed the job above.
+          exit 0

competition/LEADERBOARD.md

@@ -10,7 +10,7 @@ payout rank is settled by the monthly sealed commit-reveal run.
 
 | # | champion | PR | dethroned on | scored mean | margin over prior |
 |---|----------|----|--------------|-------------|-------------------|
-| 0 | baseline kit (repo defaults) | — | 2026-07-27 | 0.57 ± 0.04 (seeds 1–6) | — |
+| 0 | baseline kit (repo defaults) | — | 2026-07-28 | 0.52 ± 0.03 (seeds 1–6) | — |
 
 payouts follow the season shares in docs/vouchbench-seasons.md
 (65/14/10/7/4). days-on-throne accrue between dethrones; the monthly

contrib/strategies/README.md

@@ -0,0 +1,58 @@
+# contrib/strategies - the engine-submission lane
+
+this is the ditto-equivalent lane: you submit **real ranking code**, not a
+config file. a strategy decides the order the reader sees retrieved
+candidates in - the place where a new fusion, a learned reranker, or a novel
+signal actually lives.
+
+## the contract
+
+your file exposes a `rank` function (or a `STRATEGY` object with a `.rank`
+method):
+
+```python
+from vouch.strategy import Candidate
+
+def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]:
+    # return candidate ids, best first
+    ...
+```
+
+- a `Candidate` has `kind`, `id`, `summary`, `score` - **data only**. your
+  code never gets the KB, the filesystem, or the network.
+- ordering is authoritative but bounded: ids you invent are ignored, and any
+  candidate you omit is appended in its original order. you can reorder and
+  de-prioritise; you cannot fabricate or hide a result.
+- [`baseline.py`](./baseline.py) is the reigning champion (returns the
+  backend order unchanged). [`example_lexical.py`](./example_lexical.py) is a
+  worked example you can study and beat.
+
+## how it is scored
+
+open a PR that touches **only** your new file under `contrib/strategies/`.
+the `koth-engine-gate` workflow:
+
+1. runs your code in a locked-down `python -I` sandbox (resource limits + an
+   audit hook that blocks network, subprocess, and filesystem writes);
+2. scores vouchbench with your strategy vs the reigning champion, paired over
+   the day's seeds;
+3. posts the scorecard and updates the engine leaderboard on a win.
+
+## the one hard rule: engine code is never auto-merged
+
+the config (kit) lane auto-merges because a kit cannot execute. **strategy
+code can**, and vouch is a library people install - so a winning strategy is
+never merged automatically. it earns the leaderboard place and the payout,
+and ships only after a human reviews the code and merges it as a new default.
+that human review is vouch's version of ditto's tee-plus-deployment gate: the
+benchmark decides the *rank*, a person decides what *ships*.
+
+reproduce any score locally:
+
+```bash
+pip install -e .
+python .github/scripts/score_strategy.py \
+  --champion contrib/strategies/baseline.py \
+  --challenger contrib/strategies/example_lexical.py \
+  --base-sha "$(git rev-parse origin/main)" --date "$(date -u +%F)"
+```

contrib/strategies/baseline.py

@@ -0,0 +1,19 @@
+"""The reigning engine-lane champion: trust the backend's fused order.
+
+This is the strategy every submission must beat. It returns the candidates in
+exactly the order retrieval handed them over - i.e. it does nothing - so a
+challenger only dethrones it by making vouch's benchmark score genuinely
+higher, not by accident.
+
+A strategy is real ranking code. It receives the query and the retrieved
+candidates (data only - no KB, no disk, no network) and returns the ids in the
+order the reader should see them. Ordering is authoritative but bounded: ids
+you invent are ignored, and any candidate you drop is appended at the tail, so
+you can reorder and de-prioritise but never fabricate or hide a result.
+"""
+
+from vouch.strategy import Candidate
+
+
+def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]:
+    return [c.id for c in candidates]

contrib/strategies/example_lexical.py

@@ -0,0 +1,34 @@
+"""A worked example: re-rank by lexical overlap with the query.
+
+Not necessarily a winner - it exists to show the shape of a real submission.
+It boosts candidates whose summary shares more words with the query, blended
+with the backend's own score so a strong retrieval signal is not thrown away.
+
+Copy this file, change the scoring, and open a PR that touches only your new
+file under contrib/strategies/. The engine gate scores it in a sandbox against
+the reigning champion; you never edit the engine itself.
+"""
+
+import re
+
+from vouch.strategy import Candidate
+
+_WORD = re.compile(r"[a-z0-9]+")
+
+
+def _tokens(text: str) -> set[str]:
+    return set(_WORD.findall(text.lower()))
+
+
+def rank(query: str, candidates: list[Candidate], *, limit: int) -> list[str]:
+    q = _tokens(query)
+    if not q:
+        return [c.id for c in candidates]
+
+    def blended(c: Candidate) -> float:
+        overlap = len(q & _tokens(c.summary)) / len(q)
+        # keep the backend score in the mix; overlap only tips ties and
+        # rescues a lexically-obvious hit the fusion under-ranked.
+        return 0.7 * c.score + 0.3 * overlap
+
+    return [c.id for c in sorted(candidates, key=blended, reverse=True)]