diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 1f610609..fc2ec411 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -104,6 +104,12 @@ "source": "./plugins/bitwarden-testing-tools", "version": "1.0.0", "description": "Testing tools for Bitwarden — analyzing and improving test quality across its repositories." + }, + { + "name": "bitwarden-doc-parity", + "source": "./plugins/bitwarden-doc-parity", + "version": "1.0.0", + "description": "Enforces the documentation standard's base obligations — docs and diagrams update in the same change as the code they describe. Session instructions, a Stop-hook tripwire, a semantic verification skill, and a CI review face." } ] } diff --git a/.cspell.json b/.cspell.json index 1d9466ad..ff8500be 100644 --- a/.cspell.json +++ b/.cspell.json @@ -31,6 +31,7 @@ "cvss", "Dashlane", "dast", + "diceware", "docstrings", "dread", "duedate", @@ -97,6 +98,7 @@ "pushback", "pyproject", "pytest", + "rawfile", "remotelink", "Rescope", "resolutiondate", diff --git a/README.md b/README.md index bccc005f..1d0d417c 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ A curated collection of plugins for AI-assisted development at Bitwarden. Enable | [bitwarden-designer](plugins/bitwarden-designer/) | 0.1.0 | Product designer persona: Code of Conduct and 30/60/90 critique, critique facilitation; dispatches into bitwarden-design-tools | | [bitwarden-design-tools](plugins/bitwarden-design-tools/) | 0.1.0 | Design toolkit: content style guide, Figma Dev Mode MCP, Bitwarden brand application, handoff prep, Design System governance, Product and Design Jira | | [bitwarden-devops-engineer](plugins/bitwarden-devops-engineer/) | 0.1.5 | DevOps engineering assistant: workflow compliance linting, action security auditing, and org-wide CI/CD remediation | +| [bitwarden-doc-parity](plugins/bitwarden-doc-parity/) | 1.0.0 | Keeps docs current with the code they describe: session instructions, a Stop-hook tripwire, a semantic verification skill, and a CI review face | | [bitwarden-init](plugins/bitwarden-init/) | 1.2.2 | Initialize and enhance CLAUDE.md files with Bitwarden's standardized template format | | [bitwarden-product-analyst](plugins/bitwarden-product-analyst/) | 0.1.5 | Product analyst agent for creating comprehensive Bitwarden requirements documents from multiple sources | | [bitwarden-security-engineer](plugins/bitwarden-security-engineer/) | 1.3.0 | Application security engineering: vulnerability triage, threat modeling, and secure code analysis | diff --git a/plugins/bitwarden-doc-parity/.claude-plugin/plugin.json b/plugins/bitwarden-doc-parity/.claude-plugin/plugin.json new file mode 100644 index 00000000..5f39ff71 --- /dev/null +++ b/plugins/bitwarden-doc-parity/.claude-plugin/plugin.json @@ -0,0 +1,19 @@ +{ + "name": "bitwarden-doc-parity", + "version": "1.0.0", + "description": "Enforces the documentation standard's base obligations — docs and diagrams update in the same change as the code they describe. Session instructions, a Stop-hook tripwire, a semantic verification skill, and a CI review face.", + "author": { + "name": "Bitwarden", + "url": "https://github.com/bitwarden" + }, + "homepage": "https://github.com/bitwarden/ai-plugins/tree/main/plugins/bitwarden-doc-parity", + "repository": "https://github.com/bitwarden/ai-plugins", + "keywords": [ + "documentation", + "doc-parity", + "doc-drift", + "hooks", + "standards", + "review" + ] +} diff --git a/plugins/bitwarden-doc-parity/CHANGELOG.md b/plugins/bitwarden-doc-parity/CHANGELOG.md new file mode 100644 index 00000000..a9a6be6d --- /dev/null +++ b/plugins/bitwarden-doc-parity/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog + +All notable changes to the bitwarden-doc-parity plugin will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.0.0] - 2026-08-04 + +### Added + +- Initial release of the `bitwarden-doc-parity` plugin, enforcing the documentation standard's base obligations to monitor for drift between code and documentation. Operates within-repo only when fired as a stop-hook and includes out-of-repo validation against contributing-docs when being executed in a PR review. +- Behavior evals for `verifying-doc-parity` against the bitwarden/server Seeder subsystem as the first ground-truth corpus. diff --git a/plugins/bitwarden-doc-parity/README.md b/plugins/bitwarden-doc-parity/README.md new file mode 100644 index 00000000..29109bc8 --- /dev/null +++ b/plugins/bitwarden-doc-parity/README.md @@ -0,0 +1,51 @@ +# bitwarden-doc-parity + +Enforces the [documentation standard's](https://contributing.bitwarden.com/contributing/documentation) requirement that docs and diagrams update in the same change as the code they describe. Changes that invalidate out-of-repo docs get called out at review. + +## Design + +Four layers. Each layer covers the weakness of the one before it. + +### Layer 1: instruction fragment + +A `SessionStart` hook injects the [base documentation obligations](./hooks/doc-parity-instructions.md) into every session. + +The fragment closes with a pointer to the published standard. This layer is prevention, so most sessions never reach the gate. + +### Layer 2: Stop hook + +A [deterministic tripwire](./hooks/doc-parity-check.sh) on the session-end event. The hook's scope is in-repo documentation only, since a file-path tripwire can only see the tree; out-of-repo docs belong to the CI face. + +The hook blocks once per session and its message directs the agent to run the verification skill. + +### Layer 3: semantic verification skill + +`verifying-doc-parity` is the intelligence the hook invokes, also invocable on demand: read the diff, then read the documentation at every documented ancestor of the change, since references may occur at any level. + +The outcome, per documented scope, is either an update or an explicit attestation that nothing documented there drifted. + +### Layer 4: CI face + +The same skill logic run as a PR reviewer through the consuming repository's ai-review workflow, covering human-authored changes that no in-session layer sees. This layer also owns out-of-repo discovery. Search terms are derived from the diff and used to search contributing-docs for references that need to be addressed. The callout triggers the standard's external-docs flow: a work item before merge and a stale marker on the page. + +This layer is inert until the review pipeline invokes `verifying-doc-parity`. When installed alongside `bitwarden-code-review` (from 1.14.0), the reviewer agent's Cross-Plugin Enrichment picks the skill up automatically. If the consuming repository uses a different review workflow, wire the skill into that workflow's documentation pass. + +## Installation + +```bash +/plugin install bitwarden-doc-parity@bitwarden-marketplace +``` + +Restart Claude Code after installing so the hooks load. + +## Usage + +The plugin needs no invocation in normal work: the fragment loads at session start, and the Stop hook fires only when the trigger rule matches. When the hook blocks, follow its message and run the skill. To verify documentation parity on demand: + +```text +Do I need to update any docs for my current changes? +``` + +## Requirements + +- `bash`, `git`, and `jq` on `PATH`. The hooks fail open: when a requirement is missing or the working directory is not a git repository, sessions proceed unaffected. diff --git a/plugins/bitwarden-doc-parity/hooks/doc-parity-check.sh b/plugins/bitwarden-doc-parity/hooks/doc-parity-check.sh new file mode 100755 index 00000000..5e68496d --- /dev/null +++ b/plugins/bitwarden-doc-parity/hooks/doc-parity-check.sh @@ -0,0 +1,151 @@ +#!/bin/bash +# doc-parity-check.sh +# Stop hook: deterministic tripwire for stale in-repo documentation. +# +# Trigger rule: fire when a changed file has a documented ancestor scope +# below the repo root and no documentation along its ancestor chain was +# touched. A directory is a documented scope when it contains a README.md +# or a docs/ directory. Root-scope docs do not arm the tripwire, since the +# root is an ancestor of every file; their verification belongs to the +# semantic layer. +# +# Behavior: blocks exactly once per session with a message directing the +# agent to the verifying-doc-parity skill, then allows through. +# Generalizes bitwarden/server's .claude/hooks/seeder-docs-check.sh. +# +# Fail-open: any environment problem (no jq, no git repo) lets the turn end without blocking. + +set -uo pipefail + +command -v jq >/dev/null 2>&1 || exit 0 + +INPUT=$(cat) + +# Guard: if a Stop hook already blocked this turn, allow through. +STOP_HOOK_ACTIVE=$(echo "$INPUT" | jq -r '.stop_hook_active // false') +if [[ "$STOP_HOOK_ACTIVE" == "true" ]]; then + exit 0 +fi + +# Guard: block at most once per session. Markers live under a plugin-owned +# subdirectory of TMPDIR to keep them off the world-writable top level, and +# the -O check skips markers we do not own so a stray file cannot silently +# suppress the session's only block. +SESSION_ID=$(echo "$INPUT" | jq -r '.session_id // empty' | tr -cd 'a-zA-Z0-9_-') +MARKER="" +if [[ -n "$SESSION_ID" ]]; then + MARKER_DIR="${TMPDIR:-/tmp}/doc-parity" + mkdir -p "$MARKER_DIR" 2>/dev/null || true + MARKER="${MARKER_DIR}/blocked-${SESSION_ID}" + if [[ -f "$MARKER" && -O "$MARKER" ]]; then + exit 0 + fi +fi + +CWD=$(echo "$INPUT" | jq -r '.cwd // empty') +if [[ -z "$CWD" ]]; then + exit 0 +fi + +REPO_ROOT=$(git -C "$CWD" rev-parse --show-toplevel 2>/dev/null) || exit 0 + +# Gather all changed files (staged, unstaged, and untracked) relative to repo root. +# core.quotePath=false so non-ASCII filenames come through unquoted and match the +# filesystem checks below. +DIFF_HEAD=$(git -C "$REPO_ROOT" -c core.quotePath=false diff --name-only HEAD 2>/dev/null || true) +UNTRACKED=$(git -C "$REPO_ROOT" -c core.quotePath=false ls-files --others --exclude-standard 2>/dev/null || true) +ALL_CHANGED=$(printf "%s\n%s" "$DIFF_HEAD" "$UNTRACKED" | sort -u | grep -v '^$' || true) + +if [[ -z "$ALL_CHANGED" ]]; then + exit 0 +fi + +# A file is documentation when it is a markdown or diagram source file, or +# when it lives under a docs/ directory. +is_doc_file() { + case "$1" in + *.md | *.mdx | *.mmd | *.mermaid) return 0 ;; + docs/* | */docs/*) return 0 ;; + esac + return 1 +} + +# The scope a documentation file describes: the directory holding it, or the +# parent of its docs/ directory when it lives under one. +doc_scope() { + local path="$1" + case "$path" in + docs/*) echo "." ;; + */docs/*) echo "${path%%/docs/*}" ;; + *) dirname "$path" ;; + esac +} + +# Partition the change set and collect the scope of every touched doc. +CHANGED_CODE="" +TOUCHED_DOC_SCOPES="" +while IFS= read -r file; do + if is_doc_file "$file"; then + TOUCHED_DOC_SCOPES="${TOUCHED_DOC_SCOPES}$(doc_scope "$file")"$'\n' + else + CHANGED_CODE="${CHANGED_CODE}${file}"$'\n' + fi +done <<<"$ALL_CHANGED" + +if [[ -z "$CHANGED_CODE" ]]; then + exit 0 +fi +TOUCHED_DOC_SCOPES=$(printf '%s' "$TOUCHED_DOC_SCOPES" | sort -u) + +# A touched doc covers a file when the doc's scope is an ancestor of the file. +is_covered() { + local file="$1" scope + while IFS= read -r scope; do + [[ -z "$scope" ]] && continue + if [[ "$file" == "$scope/"* ]]; then + return 0 + fi + done <<<"$TOUCHED_DOC_SCOPES" + return 1 +} + +# Documented ancestor scopes of a file, strictly below the repo root. +# Match README case-insensitively; CLAUDE.md and docs/ are the other markers, +# aligning with SKILL.md's scope definition. +documented_ancestors() { + local dir + dir=$(dirname "$1") + while [[ "$dir" != "." && "$dir" != "/" ]]; do + if compgen -G "$REPO_ROOT/$dir/[Rr][Ee][Aa][Dd][Mm][Ee].md" >/dev/null 2>&1 \ + || [[ -f "$REPO_ROOT/$dir/CLAUDE.md" || -d "$REPO_ROOT/$dir/docs" ]]; then + echo "$dir" + fi + dir=$(dirname "$dir") + done +} + +VIOLATIONS="" +VIOLATION_SCOPES="" +while IFS= read -r file; do + [[ -z "$file" ]] && continue + ancestors=$(documented_ancestors "$file") + [[ -z "$ancestors" ]] && continue + if ! is_covered "$file"; then + VIOLATIONS="${VIOLATIONS} - ${file}"$'\n' + VIOLATION_SCOPES="${VIOLATION_SCOPES}${ancestors}"$'\n' + fi +done <<<"$CHANGED_CODE" + +if [[ -z "$VIOLATIONS" ]]; then + exit 0 +fi + +VIOLATION_SCOPES=$(printf '%s' "$VIOLATION_SCOPES" | sort -u | sed 's/^/ - /') + +if [[ -n "$MARKER" ]]; then + touch "$MARKER" 2>/dev/null || true +fi + +REASON=$(printf 'Code changed inside documented scopes, but no documentation along the changed files'\'' ancestor chains was touched.\n\nChanged files without a documentation update:\n%s\nDocumented scopes involved:\n%s\n\nRun the bitwarden-doc-parity:verifying-doc-parity skill now: read the diff, then verify or update the documentation at every documented ancestor scope of the change. If nothing documented at a scope drifted, say so explicitly per scope. Do not make a token documentation edit to satisfy this check.' "$VIOLATIONS" "$VIOLATION_SCOPES") + +jq -n --arg reason "$REASON" '{ "decision": "block", "reason": $reason }' diff --git a/plugins/bitwarden-doc-parity/hooks/doc-parity-context.sh b/plugins/bitwarden-doc-parity/hooks/doc-parity-context.sh new file mode 100755 index 00000000..07dad37f --- /dev/null +++ b/plugins/bitwarden-doc-parity/hooks/doc-parity-context.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# doc-parity-context.sh +# SessionStart hook: injects the base documentation obligations into every +# session as additional context, so no repo has to hand-carry them in its +# root CLAUDE.md. The canonical obligation text lives in +# doc-parity-instructions.md next to this script. +# +# Fail-open: if the fragment or jq is unavailable, the session proceeds +# without the context rather than erroring. + +set -uo pipefail + +FRAGMENT="${CLAUDE_PLUGIN_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}/hooks/doc-parity-instructions.md" + +if [[ ! -f "$FRAGMENT" ]] || ! command -v jq >/dev/null 2>&1; then + exit 0 +fi + +jq -n --rawfile ctx "$FRAGMENT" '{ + hookSpecificOutput: { + hookEventName: "SessionStart", + additionalContext: $ctx + } +}' diff --git a/plugins/bitwarden-doc-parity/hooks/doc-parity-instructions.md b/plugins/bitwarden-doc-parity/hooks/doc-parity-instructions.md new file mode 100644 index 00000000..939b8c0c --- /dev/null +++ b/plugins/bitwarden-doc-parity/hooks/doc-parity-instructions.md @@ -0,0 +1,5 @@ +# Documentation parity obligations + +Before modifying code, read in-code documentation associated with it, any `README.md` or `docs/`, and any diagrams that may reference it. When you change code that has associated documentation or diagrams, update that documentation in the same change. If behavior contradicts documentation and the code is correct, fix the documentation. Do not leave them disagreeing. + +These obligations come from the documentation standard: https://contributing.bitwarden.com/contributing/documentation diff --git a/plugins/bitwarden-doc-parity/hooks/hooks.json b/plugins/bitwarden-doc-parity/hooks/hooks.json new file mode 100644 index 00000000..5534b2b5 --- /dev/null +++ b/plugins/bitwarden-doc-parity/hooks/hooks.json @@ -0,0 +1,26 @@ +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/doc-parity-context.sh\"", + "timeout": 5 + } + ] + } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/doc-parity-check.sh\"", + "timeout": 15 + } + ] + } + ] + } +} diff --git a/plugins/bitwarden-doc-parity/skills/verifying-doc-parity/SKILL.md b/plugins/bitwarden-doc-parity/skills/verifying-doc-parity/SKILL.md new file mode 100644 index 00000000..1761dd31 --- /dev/null +++ b/plugins/bitwarden-doc-parity/skills/verifying-doc-parity/SKILL.md @@ -0,0 +1,69 @@ +--- +name: verifying-doc-parity +description: This skill should be used when the user mentions doc drift, documentation verification, README or docs/ updates that follow a code change, or a doc-parity Stop hook block — even if the request does not name a skill or documentation explicitly. Also runs as the documentation pass of a pull request review. Triggered by phrasings such as "verify doc parity", "are my docs up to date", "did I miss any doc updates", "what documentation should this change touch", "do I need to update any docs for my current changes", or "check if the docs still match the code". +agent: general-purpose +context: fork +background: false +allowed-tools: WebFetch(domain:contributing.bitwarden.com) +--- + +# Verifying documentation parity + +This skill is the judgment that code has drifted away from the documentation which describes it. It enforces the base obligations of the [documentation standard](https://contributing.bitwarden.com/contributing/documentation): docs and diagrams update in the same change as the code they describe, and a change that invalidates documentation the repo does not contain gets called out at review. + +## Contexts + +The skill runs in two contexts where change source and relevant document discovery differ. + +- **Agent session**, invoked by the doc-parity Stop hook or on demand. The change is the working tree (`git diff HEAD` plus untracked files). Verification is in-repo only. +- **Pull request review**, run through the ai-review workflow. The change is the PR diff plus the PR description. In addition to in-repo verification, this context performs [out-of-repo discovery](#out-of-repo-discovery-review-context-only). + +## Workflow + +### Step 1: Assemble the change + +Collect the full set of changed files and read the diff, not just the file list, because judging drift requires knowing what the change does. In a session, use `git diff HEAD` and `git ls-files --others --exclude-standard`. In a review, use the PR diff and read the PR description for intent. + +Before walking the tree, consider whether the change qualifies as a tripwire false positive: a formatting-only diff, generated output, or a doc-comment-only edit that the Stop hook classified as code. When it does, **dismiss** the check with a one-line reason naming the false-positive class and end the skill. + +### Step 2: Enumerate every documented ancestor scope + +For each changed file, walk its directory chain from the file's own directory up to and including the repo root. A directory is a documented scope when it contains a `README.md`, a `docs/` directory, or diagram sources. Below component scope, the documentation surface is source-embedded: Rust `//!` module docs, C# XML doc comments, JSDoc/TSDoc on public symbols. When the changed area of a source file carries such doc comments, treat that file as its own documented surface. Collect the union of documented scopes and surfaces across all changed files. + +Check every documented ancestor, not just the nearest one, since documentation layers by altitude: higher views sand off detail while still describing the changed behavior. A change can be current in its component README yet drift a root-level guide or a container diagram two scopes up. + +### Step 3: Judge and act, per scope + +For each documented scope or surface, exactly one of two outcomes: + +- **Update.** If behavior contradicts documentation and the code is correct, fix the documentation. Do not leave them disagreeing. If the change adds behavior this scope's altitude should describe, add documentation of that new behavior. A scope's documentation describes what is present at that scope and below, regardless of whether higher-level callers currently exercise or guard against its use. Edit the documentation in the same change. When code was removed, remove its documentation, and treat a moved doc as a strict move. Every edit conforms to the [documentation standard](https://contributing.bitwarden.com/contributing/documentation) — its placement rule, its style guide, and any repo-local guidance layered on top. Consult the standard when a placement, format, or style question isn't obvious from what you already have. If placement routes a doc outside the working repo, handle it as an out-of-repo callout (see below) rather than an in-repo edit. +- **Attest.** Nothing documented at this scope drifted. State that explicitly, with a one-line reason grounded in what the doc actually says. + +### Step 4: Report with per-scope attestation + +Close with an explicit per-scope list so the user, or the PR review summary, can audit the judgment: + +```text +Documentation parity: +- util/Seeder/Data (README.md) — updated: generator table gained the new distribution. +- util/Seeder (README.md) — verified current: the change does not alter the encryption axes the README describes. +- crates/bitwarden-crypto/src/lib.rs (module docs) — verified current: the `derive_`/`make_` naming invariants still hold. +- repo root (README.md) — verified current: no root-level behavior described there changed. +``` + +Every documented scope and surface from Step 2 appears in the list. A scope missing from the list means the verification is incomplete. + +When the check exits early via Dismiss, the report is a single line naming the false-positive class: + +```text +Documentation parity: dismissed — formatting-only diff, no documentation obligation. +``` + +## Out-of-repo discovery (review context only) + +Out-of-repo documentation cannot be found by walking the tree, so derive search angles from the change itself: + +1. Derive search terms from the diff: changed paths, public symbols, message and endpoint names, feature vocabulary from the PR description, explicit external links. +2. Search the contributing-docs (contributing.bitwarden.com, source repo `bitwarden/contributing-docs`) from those angles. +3. Read the candidate pages and judge, as in Step 3, whether the change invalidates them. +4. Call out every invalidated page in the review. The callout triggers the standard's external-docs flow per the documentation standard. diff --git a/plugins/bitwarden-doc-parity/skills/verifying-doc-parity/evals/README.md b/plugins/bitwarden-doc-parity/skills/verifying-doc-parity/evals/README.md new file mode 100644 index 00000000..cd78e530 --- /dev/null +++ b/plugins/bitwarden-doc-parity/skills/verifying-doc-parity/evals/README.md @@ -0,0 +1,48 @@ +# verifying-doc-parity evals + +Behavior test cases for the `verifying-doc-parity` skill, in the `skill-creator` schema. + +`behavior-eval.json` holds cases exercising what happens when the skill fires. Covers both primary success paths (session and pull-request review), every outcome the skill claims to produce (Update, Attest, Dismiss), and each recently-added instruction (below-component source-embedded surfaces, placement follows what the doc describes, root-level placement for a repo-wide capability). + +`behavior-baseline.json` is the recorded benchmark that intentional skill changes must not regress silently. + +## Do not add trigger evals for this skill + +It was found that the `run_eval.py`'s methodology incorrectly failed to detect skill triggers due to requiring immediate skill execution rather than context exploration, which models almost always do regardless of description changes. + +This skill's load-bearing invocation paths are the doc-parity Stop hook and the ai-review workflow's documentation pass. Natural-language triggering is a nice-to-have fallback, not the critical path. + +## Eval cases + +Behavior cases run against the Seeder subsystem of [bitwarden/server](https://github.com/bitwarden/server) (`util/Seeder/`, `util/SeederApi/`, `util/SeederUtility/`), chosen because its code-to-docs mapping is known. + +These evals are executed against a pinned server ref, `d6c84a7562cc6b464de910dbf829690885500137`. To increase stability in testing, evaluate against that reference point. + +Each case is constructed so a run that skips a documented scope, stays silent instead of attesting, token-edits its way past the gate, or mishandles a specific outcome type (Update, Attest, Dismiss) fails a named expectation. Behavior cases assume the run happens inside a bitwarden/server checkout so the ground-truth docs are readable. + +Case 10 exercises the review context (out-of-repo discovery against the contributing-docs corpus); all other cases are session cases. Cases involve live edits to a scratch checkout, so run them against a disposable clone or a git worktree, never a working tree you care about. + +Each case's `expectations` array is the pass criterion — every expectation is graded independently. Denominators differ per case because they count expectations, not runs. + +## Running + +Run with `/skill-creator:skill-creator` in benchmark mode (with-skill vs. without-skill) with a config-blind grader. Install the runner from the `claude-plugins-official` marketplace if it is not already present: + +```bash +/plugin install skill-creator@claude-plugins-official +``` + +The AI Review Guidelines want at least three iterations per case per configuration so pass rate and variance are both captured. + +Regression check against the current baseline compares the aggregate summary and per-run pass rates; timestamps, models, and per-run evidence prose vary on every execution and are not signal: + +```bash +proj='{run_summary, runs: [.runs[] | {eval_id, configuration, run_number, pass_rate: .result.pass_rate}]}' +diff <(jq -S "$proj" behavior-baseline.json) <(jq -S "$proj" result.json) +``` + +An empty diff on that projection means no regression on the graded data. When a change is intentional and the new numbers are the new desired state, replace the baseline with the new results in the same PR as the skill change. + +## Updating the test surface + +Update `behavior-eval.json` when the test surface needs to evolve: a new load-bearing rule in the skill, a Seeder doc restructure that invalidates a case's ground truth, or an expectation that turned out to be ambiguous. The eval set encodes intent, not implementation — when a run fails, fix the skill, not the case, unless the case itself is wrong. diff --git a/plugins/bitwarden-doc-parity/skills/verifying-doc-parity/evals/behavior-baseline.json b/plugins/bitwarden-doc-parity/skills/verifying-doc-parity/evals/behavior-baseline.json new file mode 100644 index 00000000..a29e01ce --- /dev/null +++ b/plugins/bitwarden-doc-parity/skills/verifying-doc-parity/evals/behavior-baseline.json @@ -0,0 +1,3865 @@ +{ + "metadata": { + "skill_name": "verifying-doc-parity", + "skill_path": "plugins/bitwarden-doc-parity/skills/verifying-doc-parity", + "executor_model": "claude-opus-5", + "analyzer_model": "claude-opus-5", + "timestamp": "2026-08-06T21:46:54Z", + "evals_run": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + "runs_per_configuration": 5, + "corpus_ref": { + "repo": "bitwarden/server", + "commit": "d6c84a7562cc6b464de910dbf829690885500137" + } + }, + "runs": [ + { + "eval_id": 1, + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 0.75, + "passed": 3, + "failed": 1, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Enumerates util/Seeder/Data, util/Seeder, and the repo root as documented scopes of the change, not only the nearest README", + "passed": true, + "evidence": "Report's 'Documented scopes and surfaces enumerated' lists 'util/Seeder/Data/ \u2014 README.md', 'util/Seeder/ \u2014 README.md, CLAUDE.md', and 'repo root \u2014 README.md'." + }, + { + "text": "Reads util/Seeder/Data/README.md before judging, and updates its generator documentation to include the new generator", + "passed": false, + "evidence": "The report reads util/Seeder/Data/README.md but declines to update it ('Verified current... Listing it would misrepresent what the seeder can actually produce end-to-end today'); worktree.diff shows only the .cs change with no README update." + }, + { + "text": "For each scope with no drift, gives an explicit attestation with a reason grounded in the doc's actual content, rather than staying silent about the scope", + "passed": true, + "evidence": "Each scope has a 'Verified current' attestation grounded in doc content, e.g. util/Seeder/README.md 'explicitly defers generator/distribution details to Data/README.md' and CLAUDE.md 'Adding a new generator: 3-step pattern - followed'." + }, + { + "text": "Ends with a per-scope report in which every enumerated documented scope appears exactly once as either updated or verified current", + "passed": true, + "evidence": "Closing summary lists all six enumerated scopes/surfaces each exactly once marked 'verified current'." + } + ], + "notes": [] + }, + { + "eval_id": 1, + "configuration": "with_skill", + "run_number": 2, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Enumerates util/Seeder/Data, util/Seeder, and the repo root as documented scopes of the change, not only the nearest README", + "passed": true, + "evidence": "'Documented ancestor scopes enumerated' explicitly lists 'util/Seeder/Data/ \u2014 README.md present. Documented scope', 'util/Seeder/ \u2014 README.md and CLAUDE.md present', and 'Repo root \u2014 README.md present'." + }, + { + "text": "Reads util/Seeder/Data/README.md before judging, and updates its generator documentation to include the new generator", + "passed": true, + "evidence": "worktree.diff shows an addition to util/Seeder/Data/README.md: '+- Diceware-style passphrases via `PassphraseGenerator` (200-word pool, default 5 words joined by hyphens)'." + }, + { + "text": "For each scope with no drift, gives an explicit attestation with a reason grounded in the doc's actual content, rather than staying silent about the scope", + "passed": true, + "evidence": "Each non-updated scope has an explicit 'verified current' with grounded reasons, e.g. util/Seeder/README.md 'defers to Data/README.md for generator/distribution details' and CLAUDE.md 'Parallelism section states the general rule... does not enumerate individual lazy properties'." + }, + { + "text": "Ends with a per-scope report in which every enumerated documented scope appears exactly once as either updated or verified current", + "passed": true, + "evidence": "The 'Documentation parity' closing section lists util/Seeder/Data (updated), util/Seeder README (verified), util/Seeder CLAUDE.md (verified), repo root (verified), plus both XML surfaces \u2014 each exactly once." + } + ], + "notes": [] + }, + { + "eval_id": 1, + "configuration": "with_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Enumerates util/Seeder/Data, util/Seeder, and the repo root as documented scopes of the change, not only the nearest README", + "passed": true, + "evidence": "'Documented ancestor scopes' bullets list util/Seeder/Data/README.md, util/Seeder/README.md, util/Seeder/CLAUDE.md, and repo root README.md as documented scopes." + }, + { + "text": "Reads util/Seeder/Data/README.md before judging, and updates its generator documentation to include the new generator", + "passed": true, + "evidence": "worktree.diff adds '+- Diceware-style passphrase generation (hyphen-joined nouns, default 5 words)' to util/Seeder/Data/README.md's Current Capabilities > Login Ciphers list." + }, + { + "text": "For each scope with no drift, gives an explicit attestation with a reason grounded in the doc's actual content, rather than staying silent about the scope", + "passed": true, + "evidence": "Every non-updated scope has an attestation grounded in what the doc actually says, e.g. util/Seeder/README.md 'delegates to Data/README.md for generator details' and CLAUDE.md's 'New files under Data/ belong in the matching subfolder' rule check." + }, + { + "text": "Ends with a per-scope report in which every enumerated documented scope appears exactly once as either updated or verified current", + "passed": true, + "evidence": "'Per-scope attestation' section closes with all six enumerated scopes/surfaces each once \u2014 Data/README.md 'updated', others 'verified current'." + } + ], + "notes": [] + }, + { + "eval_id": 1, + "configuration": "with_skill", + "run_number": 4, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Enumerates util/Seeder/Data, util/Seeder, and the repo root as documented scopes of the change, not only the nearest README", + "passed": true, + "evidence": "Step 2 lists util/Seeder/Data (README.md), util/Seeder (README.md and CLAUDE.md), and Repo root (README.md) as 'Documented scope'." + }, + { + "text": "Reads util/Seeder/Data/README.md before judging, and updates its generator documentation to include the new generator", + "passed": true, + "evidence": "worktree.diff adds '+- Diceware-style passphrases (`PassphraseGenerator`) from a 200-word seeded pool, hyphen-joined, default 5 words' to util/Seeder/Data/README.md's Login Ciphers section." + }, + { + "text": "For each scope with no drift, gives an explicit attestation with a reason grounded in the doc's actual content, rather than staying silent about the scope", + "passed": true, + "evidence": "Each 'verified current' scope cites the doc content, e.g. util/Seeder/README.md 'explicitly delegates to Data/README.md for Generators/Distributions detail' and CLAUDE.md's 'Data/ File Organization' rule + thread-safety guidance." + }, + { + "text": "Ends with a per-scope report in which every enumerated documented scope appears exactly once as either updated or verified current", + "passed": true, + "evidence": "Step 5 'Per-scope attestation' lists all six enumerated scopes/surfaces each exactly once as updated or verified current." + } + ], + "notes": [] + }, + { + "eval_id": 1, + "configuration": "with_skill", + "run_number": 5, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Enumerates util/Seeder/Data, util/Seeder, and the repo root as documented scopes of the change, not only the nearest README", + "passed": true, + "evidence": "The 'Ancestor scopes and surfaces enumerated' table marks util/Seeder/Data (README.md), util/Seeder (README.md and CLAUDE.md), and repo root (README.md) as documented scopes." + }, + { + "text": "Reads util/Seeder/Data/README.md before judging, and updates its generator documentation to include the new generator", + "passed": true, + "evidence": "worktree.diff adds '+- Diceware-style passphrase generation (200-word deterministic pool, 5-word default, hyphen-joined)' to util/Seeder/Data/README.md under Current Capabilities > Login Ciphers." + }, + { + "text": "For each scope with no drift, gives an explicit attestation with a reason grounded in the doc's actual content, rather than staying silent about the scope", + "passed": true, + "evidence": "The 'Attestations' section grounds each in doc content, e.g. util/Seeder/README.md 'does not enumerate individual generators, and the change adds neither a new pattern nor a new encryption axis' and CLAUDE.md's Data/ File Organization + thread-safety rules." + }, + { + "text": "Ends with a per-scope report in which every enumerated documented scope appears exactly once as either updated or verified current", + "passed": true, + "evidence": "The closing 'Documentation parity' section lists all six enumerated scopes/surfaces each exactly once as 'updated' or 'verified current'." + } + ], + "notes": [] + }, + { + "eval_id": 10, + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Derives search terms from the renamed command verbs, changed paths, and the PR title vocabulary", + "passed": true, + "evidence": "Report lists concrete search queries derived from the diff: 'site:contributing.bitwarden.com SeederUtility preset', 'site:contributing.bitwarden.com seeder preset --name', 'contributing.bitwarden.com seeder database test data' \u2014 covering the utility name, the renamed verb, and both old/new flag vocabulary." + }, + { + "text": "Searches the contributing-docs corpus (contributing.bitwarden.com / bitwarden/contributing-docs) with those terms and reads candidate pages before judging", + "passed": true, + "evidence": "Report states 'Searched contributing.bitwarden.com for SeederUtility and seeder preset CLI content' with three explicit queries, and characterizes the content it found: 'the contributing docs cover Infrastructure.IntegrationTest / EF integration testing, not the dev/seed.ps1 + SeederUtility CLI flow' \u2014 showing candidate pages were consulted." + }, + { + "text": "For any external page the rename invalidates, calls it out and states the required external-docs flow: a work item created before merge and a stale marker applied to the page", + "passed": true, + "evidence": "Report concludes 'No external Bitwarden documentation page describes the SeederUtility preset --name invocation, so there is no external-docs callout to raise from this diff.' \u2014 vacuously satisfied because no page was invalidated." + }, + { + "text": "Also verifies the in-repo documented scopes, including cross-references from util/Seeder/CLAUDE.md to util/SeederUtility", + "passed": true, + "evidence": "Report cites util/Seeder/CLAUDE.md line 183 explicitly: 'When CLI flags, commands, or preset names change, scan all *.md files under Seeds/ and SeederUtility/ for stale references' and lists CLAUDE.md in the per-scope report as verified current, then acts on that rule with fan-out edits." + } + ], + "notes": [] + }, + { + "eval_id": 10, + "configuration": "with_skill", + "run_number": 2, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Derives search terms from the renamed command verbs, changed paths, and the PR title vocabulary", + "passed": true, + "evidence": "Report explicitly enumerates 'Search terms derived from the diff: SeederUtility, preset, --name, --preset-name' \u2014 combining the utility name, verb, and old/new flag vocabulary." + }, + { + "text": "Searches the contributing-docs corpus (contributing.bitwarden.com / bitwarden/contributing-docs) with those terms and reads candidate pages before judging", + "passed": true, + "evidence": "Report says 'Searched contributing.bitwarden.com and the bitwarden/contributing-docs GitHub repo via WebSearch, and fetched the server setup guide (getting-started/server/guide/) via WebFetch' \u2014 both search platforms hit and a specific candidate page fetched and read." + }, + { + "text": "For any external page the rename invalidates, calls it out and states the required external-docs flow: a work item created before merge and a stale marker applied to the page", + "passed": true, + "evidence": "Report concludes 'No page in the contributing docs references the SeederUtility preset --name flag. No external callouts needed.' \u2014 vacuously satisfied because no external page was invalidated." + }, + { + "text": "Also verifies the in-repo documented scopes, including cross-references from util/Seeder/CLAUDE.md to util/SeederUtility", + "passed": true, + "evidence": "Report explicitly cites 'per util/Seeder/CLAUDE.md lines 179-186 under Maintenance rules: \"When CLI flags, commands, or preset names change, scan all *.md files under Seeds/ and SeederUtility/ for stale references.\" This explicitly extends the scope of the check to the sibling util/Seeder/Seeds/ tree.'" + } + ], + "notes": [] + }, + { + "eval_id": 10, + "configuration": "with_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Derives search terms from the renamed command verbs, changed paths, and the PR title vocabulary", + "passed": true, + "evidence": "Report enumerates 'Search terms derived from the diff: SeederUtility, preset --name, preset --preset-name, seeder preset' \u2014 combining the utility name, verb, and old/new flag vocabulary." + }, + { + "text": "Searches the contributing-docs corpus (contributing.bitwarden.com / bitwarden/contributing-docs) with those terms and reads candidate pages before judging", + "passed": true, + "evidence": "Report states 'Searched contributing.bitwarden.com via web search and fetched the plausible landing page (getting-started/server/guide/). None of the indexed pages, nor the setup guide, reference the SeederUtility CLI' \u2014 candidate page fetched and read." + }, + { + "text": "For any external page the rename invalidates, calls it out and states the required external-docs flow: a work item created before merge and a stale marker applied to the page", + "passed": true, + "evidence": "Report concludes 'No out-of-repo callout needed.' and 'contributing.bitwarden.com - verified current: no indexed page describes the SeederUtility CLI or its --name flag; no external callout needed.' \u2014 vacuously satisfied because no external page was invalidated." + }, + { + "text": "Also verifies the in-repo documented scopes, including cross-references from util/Seeder/CLAUDE.md to util/SeederUtility", + "passed": true, + "evidence": "Report quotes CLAUDE.md's rule directly: 'per util/Seeder/CLAUDE.md's explicit instruction: \"When CLI flags, commands, or preset names change, scan all *.md files under Seeds/ and SeederUtility/ for stale references\"', and includes 'util/Seeder (CLAUDE.md) - verified current: the doc-scan policy itself is unchanged; its instruction to scan Seeds/ and SeederUtility/ was followed.'" + } + ], + "notes": [] + }, + { + "eval_id": 10, + "configuration": "with_skill", + "run_number": 4, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Derives search terms from the renamed command verbs, changed paths, and the PR title vocabulary", + "passed": true, + "evidence": "Report says 'Searched contributing.bitwarden.com for terms drawn from the diff (SeederUtility, preset --name, \"seeder utility\")' \u2014 search terms drawn from the changed path, the renamed verb, and the flag." + }, + { + "text": "Searches the contributing-docs corpus (contributing.bitwarden.com / bitwarden/contributing-docs) with those terms and reads candidate pages before judging", + "passed": true, + "evidence": "Report says 'Searched contributing.bitwarden.com for terms drawn from the diff (SeederUtility, preset --name, \"seeder utility\"). No public contributing-docs page describes this internal utility or its preset verb' \u2014 corpus queried with derived terms before judgment." + }, + { + "text": "For any external page the rename invalidates, calls it out and states the required external-docs flow: a work item created before merge and a stale marker applied to the page", + "passed": true, + "evidence": "Report concludes 'no external doc is invalidated by the rename. If a contributing-docs page for the seeder is added later, its author should point at --preset-name.' \u2014 vacuously satisfied because no external page was invalidated." + }, + { + "text": "Also verifies the in-repo documented scopes, including cross-references from util/Seeder/CLAUDE.md to util/SeederUtility", + "passed": true, + "evidence": "Report quotes CLAUDE.md verbatim: 'honored the explicit repo-local guidance in util/Seeder/CLAUDE.md: \"When CLI flags, commands, or preset names change, scan all *.md files under Seeds/ and SeederUtility/ for stale references.\"' \u2014 CLAUDE.md read and its cross-reference to SeederUtility applied." + } + ], + "notes": [] + }, + { + "eval_id": 10, + "configuration": "with_skill", + "run_number": 5, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Derives search terms from the renamed command verbs, changed paths, and the PR title vocabulary", + "passed": true, + "evidence": "Report enumerates 'Search angles derived from the diff: SeederUtility, preset --name, preset --preset-name, --preset-name' \u2014 utility name (changed path), verb, and old/new flag vocabulary." + }, + { + "text": "Searches the contributing-docs corpus (contributing.bitwarden.com / bitwarden/contributing-docs) with those terms and reads candidate pages before judging", + "passed": true, + "evidence": "Report says 'Searched contributing.bitwarden.com via web search and fetched the plausible landing pages (Setup Guide, Tools, Getting Started, Server Architecture). None describe the SeederUtility CLI surface' \u2014 multiple candidate pages fetched and read." + }, + { + "text": "For any external page the rename invalidates, calls it out and states the required external-docs flow: a work item created before merge and a stale marker applied to the page", + "passed": true, + "evidence": "Report concludes 'the seeder CLI is documented in-repo only, so no external page is invalidated. No external-docs callout required.' \u2014 vacuously satisfied because no external page was invalidated." + }, + { + "text": "Also verifies the in-repo documented scopes, including cross-references from util/Seeder/CLAUDE.md to util/SeederUtility", + "passed": true, + "evidence": "Report quotes CLAUDE.md: 'util/Seeder/CLAUDE.md states: \"When CLI flags, commands, or preset names change, scan all *.md files under Seeds/ and SeederUtility/ for stale references.\" That scoped the sweep.' \u2014 CLAUDE.md read and its cross-reference to SeederUtility applied." + } + ], + "notes": [] + }, + { + "eval_id": 2, + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and util/Seeder/CLAUDE.md before concluding", + "passed": true, + "evidence": "Report enumerates 'util/Seeder/ - README.md (component README) and CLAUDE.md (Claude-facing quick reference)' and quotes concrete content from both (Services description; SeederDependencies bundle + thread-safety note)." + }, + { + "text": "Makes zero documentation edits for the behavior-preserving refactor", + "passed": true, + "evidence": "worktree.diff shows only ManglerService.cs modified; report states 'Edits made: None. No documentation drifted; nothing was updated.'" + }, + { + "text": "Ends with an explicit per-scope report where each scope is marked verified current with a one-line reason grounded in what the doc actually says", + "passed": true, + "evidence": "The Closing summary lists every scope (XML docs, util/Seeder/README.md, util/Seeder/CLAUDE.md, repo root README.md) with a one-line 'verified current' reason citing what the doc says (e.g. 'Services section's description of ManglerService still matches the code')." + }, + { + "text": "Does not make a token or cosmetic edit to satisfy the hook", + "passed": true, + "evidence": "worktree.diff contains no documentation changes; report explicitly says 'No documentation drifted; nothing was updated.'" + } + ], + "notes": [] + }, + { + "eval_id": 2, + "configuration": "with_skill", + "run_number": 2, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and util/Seeder/CLAUDE.md before concluding", + "passed": true, + "evidence": "'Docs read' explicitly lists 'util/Seeder/README.md - searched for every mangl mention (lines 119, 123, 124, 162)' and 'util/Seeder/CLAUDE.md - searched for every mangl mention (lines 30, 48, 69, 78)'." + }, + { + "text": "Makes zero documentation edits for the behavior-preserving refactor", + "passed": true, + "evidence": "worktree.diff only touches ManglerService.cs; report states 'Docs edited: None. No documentation contradicts the refactored code.'" + }, + { + "text": "Ends with an explicit per-scope report where each scope is marked verified current with a one-line reason grounded in what the doc actually says", + "passed": true, + "evidence": "Closing summary lists ManglerService.cs XML docs, util/Seeder README.md, util/Seeder CLAUDE.md, and repo root README.md each 'verified current' with a one-line reason citing the doc content (e.g. 'unique prefixes to emails and strings' description matches the preserved @-split logic)." + }, + { + "text": "Does not make a token or cosmetic edit to satisfy the hook", + "passed": true, + "evidence": "Only source-code diff present; no doc file touched. Report notes 'a refactor still warrants walking the ancestor scopes to confirm no doc claims drift' rather than a cosmetic edit." + } + ], + "notes": [] + }, + { + "eval_id": 2, + "configuration": "with_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and util/Seeder/CLAUDE.md before concluding", + "passed": true, + "evidence": "Report has dedicated 'util/Seeder/README.md - Verified current' and 'util/Seeder/CLAUDE.md - Verified current' subsections quoting the Scenes bullet, Services paragraph, SeederDependencies list, and thread-safety note." + }, + { + "text": "Makes zero documentation edits for the behavior-preserving refactor", + "passed": true, + "evidence": "worktree.diff shows only ManglerService.cs change; closing states 'No documentation edits needed at any documented ancestor scope or on the source-embedded surface.'" + }, + { + "text": "Ends with an explicit per-scope report where each scope is marked verified current with a one-line reason grounded in what the doc actually says", + "passed": true, + "evidence": "Step 5 lists util/Seeder/Services (no scope), util/Seeder/README.md, util/Seeder/CLAUDE.md, util/ (no scope), repo root README.md, and ManglerService.cs XML docs, each 'verified current' with a specific reason from the doc content." + }, + { + "text": "Does not make a token or cosmetic edit to satisfy the hook", + "passed": true, + "evidence": "No doc file appears in the diff; report explicitly walks the tree instead of making a cosmetic edit ('Not a tripwire false positive... verification proceeds')." + } + ], + "notes": [] + }, + { + "eval_id": 2, + "configuration": "with_skill", + "run_number": 4, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and util/Seeder/CLAUDE.md before concluding", + "passed": true, + "evidence": "'Docs read' section explicitly lists util/Seeder/README.md (Scenes bullet + Services description) and util/Seeder/CLAUDE.md (SeederDependencies, thread-safety note, --mangle flag mention)." + }, + { + "text": "Makes zero documentation edits for the behavior-preserving refactor", + "passed": true, + "evidence": "worktree.diff only touches ManglerService.cs; report says 'Docs edited: None. The refactor is internal to a single method body plus two new private helpers.'" + }, + { + "text": "Ends with an explicit per-scope report where each scope is marked verified current with a one-line reason grounded in what the doc actually says", + "passed": true, + "evidence": "Closing summary lists ManglerService.cs (class doc), ManglerService.cs (MangleId doc), util/Seeder README.md, util/Seeder CLAUDE.md, and repo root README.md, each with a 'verified current' one-liner grounded in the doc's actual text." + }, + { + "text": "Does not make a token or cosmetic edit to satisfy the hook", + "passed": true, + "evidence": "Only the .cs refactor appears in the diff. Report explicitly notes 'the check runs rather than dismissing' and closes with 'The doc-parity hook can be cleared: this is a behavior-preserving internal refactor with no documented surface to update.'" + } + ], + "notes": [] + }, + { + "eval_id": 2, + "configuration": "with_skill", + "run_number": 5, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and util/Seeder/CLAUDE.md before concluding", + "passed": true, + "evidence": "'Docs read' section quotes util/Seeder/README.md Services subsection ('Context-aware string mangling for test isolation. Adds unique prefixes...') and util/Seeder/CLAUDE.md Parallelism note ('ManglerService and SeederContext are NOT thread-safe')." + }, + { + "text": "Makes zero documentation edits for the behavior-preserving refactor", + "passed": true, + "evidence": "worktree.diff shows only ManglerService.cs changed; report states 'Docs edited: None. The change is a pure private-helper extraction... so nothing at any documented altitude drifted.'" + }, + { + "text": "Ends with an explicit per-scope report where each scope is marked verified current with a one-line reason grounded in what the doc actually says", + "passed": true, + "evidence": "Attestations block lists ManglerService.cs XML doc surface, util/Seeder (README.md + CLAUDE.md), and repo root README.md, each 'verified current' with a specific one-liner citing doc content (e.g. Services description, thread-safety note)." + }, + { + "text": "Does not make a token or cosmetic edit to satisfy the hook", + "passed": true, + "evidence": "Only .cs code appears in the diff; report proceeds through scope enumeration rather than making any doc edit to clear the hook." + } + ], + "notes": [] + }, + { + "eval_id": 3, + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Refuses to make the requested cosmetic or token documentation edit", + "passed": true, + "evidence": "Report opens with 'I did not do that. That would be a mechanical bypass, not a verification' and the worktree.diff contains no edit to util/Seeder/README.md." + }, + { + "text": "Explains why the refusal \u2014 that a real per-scope verification is the correct path through the hook", + "passed": true, + "evidence": "Report states: 'The right action is to actually judge whether the code changes drift any documentation, and only edit docs that are genuinely stale. I ran the skill instead.'" + }, + { + "text": "Performs the actual verification: reads the docs at every documented ancestor scope of util/Seeder/Recipes and either updates or attests per scope", + "passed": true, + "evidence": "Enumerates util/Seeder/Recipes/, util/Seeder/, util/, repo root; reads util/Seeder/README.md, CLAUDE.md, verification.md, and repo README; provides per-scope judgment (updated vs verified current)." + }, + { + "text": "Ends with a per-scope report grounded in what the docs say", + "passed": true, + "evidence": "Closing 'Documentation parity:' list gives each scope a one-line reason: 'util/Seeder/README.md \u2014 verified current: describes Recipes at architecture altitude (BulkCopy, single entry point, Factory composition, naming)'." + } + ], + "notes": [] + }, + { + "eval_id": 3, + "configuration": "with_skill", + "run_number": 2, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Refuses to make the requested cosmetic or token documentation edit", + "passed": true, + "evidence": "Report says 'You asked me to add a trailing newline to util/Seeder/README.md to appease the Stop hook. I did not do that.' The diff contains no README.md edit." + }, + { + "text": "Explains why the refusal \u2014 that a real per-scope verification is the correct path through the hook", + "passed": true, + "evidence": "'Silencing the hook with a cosmetic edit would defeat the check and leave documentation drifting behind the code. Instead I ran the actual doc-parity verification against the real staged diff and fixed the drift I found.'" + }, + { + "text": "Performs the actual verification: reads the docs at every documented ancestor scope of util/Seeder/Recipes and either updates or attests per scope", + "passed": true, + "evidence": "Step 2 enumerates util/Seeder/Recipes/, util/Seeder/, util/, repo root; reads util/Seeder/README.md, CLAUDE.md, Data/README.md, repo README and CONTRIBUTING; updates one XML doc and attests each scope." + }, + { + "text": "Ends with a per-scope report grounded in what the docs say", + "passed": true, + "evidence": "Step 5 'Per-scope attestation' bullets ground each verdict in doc content, e.g., 'util/Seeder/ (CLAUDE.md) \u2014 verified current: Density Profiles section describes Steps under Steps/, not the direct-to-database Recipes'." + } + ], + "notes": [] + }, + { + "eval_id": 3, + "configuration": "with_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Refuses to make the requested cosmetic or token documentation edit", + "passed": true, + "evidence": "'The user asked me to \"just add a trailing newline to util/Seeder/README.md so the hook passes.\" I did not do that.' The diff contains no README.md edit." + }, + { + "text": "Explains why the refusal \u2014 that a real per-scope verification is the correct path through the hook", + "passed": true, + "evidence": "Explains: 'Adding a whitespace line to an unrelated README would falsely re-run the hook to a \"pass\" state without addressing any actual drift, and would leave a stale doc-comment behind. So I ran the real verification instead.'" + }, + { + "text": "Performs the actual verification: reads the docs at every documented ancestor scope of util/Seeder/Recipes and either updates or attests per scope", + "passed": true, + "evidence": "Enumerates util/Seeder/Recipes/, util/Seeder/, util/Seeder/Seeds/docs/, util/, repo root; reads Seeder README, CLAUDE.md, verification.md, architecture.md, presets.md, scenarios/, repo README, CONTRIBUTING; updates the stale XML doc." + }, + { + "text": "Ends with a per-scope report grounded in what the docs say", + "passed": true, + "evidence": "'Documentation parity' section lists each scope with grounded reasoning, e.g., 'util/Seeder/CLAUDE.md \u2014 verified current: names the direct-to-database Recipes only in the Recipe Contract; it does not describe their assignment cycles.'" + } + ], + "notes": [] + }, + { + "eval_id": 3, + "configuration": "with_skill", + "run_number": 4, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Refuses to make the requested cosmetic or token documentation edit", + "passed": true, + "evidence": "'The user's request was to add a trailing newline to util/Seeder/README.md to get the doc-parity Stop hook to pass. I did not do that.' The diff contains no README.md edit." + }, + { + "text": "Explains why the refusal \u2014 that a real per-scope verification is the correct path through the hook", + "passed": true, + "evidence": "'The hook is behaving correctly here: the staged diff is a real behavioral change, not a false positive, so the right response is verification, not a spurious edit to satisfy the hook.'" + }, + { + "text": "Performs the actual verification: reads the docs at every documented ancestor scope of util/Seeder/Recipes and either updates or attests per scope", + "passed": true, + "evidence": "Step 2 enumerates util/Seeder/Recipes/, util/Seeder/, util/, repo root; reads util/Seeder/README.md, CLAUDE.md, verification.md, repo README; provides per-scope updated/verified verdicts." + }, + { + "text": "Ends with a per-scope report grounded in what the docs say", + "passed": true, + "evidence": "'Documentation parity' section grounds each scope in doc content, e.g., 'util/Seeder (README.md) \u2014 verified current: the change does not alter any of the Seeder patterns, contracts, or encryption axes the README describes.'" + } + ], + "notes": [] + }, + { + "eval_id": 3, + "configuration": "with_skill", + "run_number": 5, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Refuses to make the requested cosmetic or token documentation edit", + "passed": true, + "evidence": "'The request was to add a trailing newline to util/Seeder/README.md so the doc-parity Stop hook stops firing. I didn't do that...' The diff contains no README.md edit." + }, + { + "text": "Explains why the refusal \u2014 that a real per-scope verification is the correct path through the hook", + "passed": true, + "evidence": "'The hook is right to flag these. Silencing it with an unrelated whitespace edit would hide drift, not resolve it. So I ran the actual verification.'" + }, + { + "text": "Performs the actual verification: reads the docs at every documented ancestor scope of util/Seeder/Recipes and either updates or attests per scope", + "passed": true, + "evidence": "Enumerates scopes in a table (Recipes/, Seeder/, util/, repo root); reads Seeder README, CLAUDE.md, verification.md, architecture.md, presets.md, scenarios/, repo README; updates the stale XML doc and attests each scope." + }, + { + "text": "Ends with a per-scope report grounded in what the docs say", + "passed": true, + "evidence": "Per-scope attestation grounds each verdict, e.g., 'util/Seeder/README.md \u2014 verified current: the \"Recipes\" section describes what a Recipe is (purpose, key characteristics, naming, single entry point) at a level that does not name assignment ratios or permission cycles'." + } + ], + "notes": [] + }, + { + "eval_id": 4, + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Names what the doc says (the deterministic claim) and what the code does (wall-clock non-determinism) as a concrete disagreement", + "passed": true, + "evidence": "Report names the Design Principles bullet ('**Deterministic.** Seeded randomness means same org ID -> same test data -> reproducible debugging.') and states 'The newly added `UsernameDistributions.Current` breaks that invariant by construction' \u2014 explicit contradiction stated." + }, + { + "text": "Either edits util/Seeder/Data/README.md's determinism claim, or surfaces the contradiction with a specific recommendation \u2014 does not silently defer", + "passed": true, + "evidence": "Diff shows the Design Principles bullet in util/Seeder/Data/README.md was edited: the '**Deterministic.**' line becomes '**Deterministic by default.** ... Distributions that intentionally sample wall-clock time (e.g., `UsernameDistributions.Current`) opt out'." + }, + { + "text": "Still completes per-scope verification for the actual UserStatusDistributions change", + "passed": true, + "evidence": "Report includes 'UserStatusDistributions.cs (XML doc) \u2014 verified current. The `` on `Realistic` was updated in the same hunk from ... to \"80% Confirmed, 8% Invited, 7% Accepted, 5% Revoked\"'." + }, + { + "text": "Ends with a per-scope report", + "passed": true, + "evidence": "The 'Per-scope judgment and action' section enumerates each documented scope (XML docs on both files, Data/README.md, Seeder/README.md, repo root) with a per-scope verdict, and the closing summary reiterates it." + } + ], + "notes": [] + }, + { + "eval_id": 4, + "configuration": "with_skill", + "run_number": 2, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Names what the doc says (the deterministic claim) and what the code does (wall-clock non-determinism) as a concrete disagreement", + "passed": true, + "evidence": "Report quotes the Design Principles bullet 'Deterministic. Seeded randomness means same org ID \u2192 same test data \u2192 reproducible debugging' and states it 'is directly contradicted by the newly-added `Current` distribution: with the same org ID, two runs a few hours apart will pick different sub-distributions'." + }, + { + "text": "Either edits util/Seeder/Data/README.md's determinism claim, or surfaces the contradiction with a specific recommendation \u2014 does not silently defer", + "passed": true, + "evidence": "Did not edit but explicitly surfaced with a specific recommendation: 'decide whether the `Current` addition is intentional. If yes, the two READMEs need explicit carve-outs on the determinism claim. If no, reverting the `UsernameDistributions.cs` hunk restores the design principle'." + }, + { + "text": "Still completes per-scope verification for the actual UserStatusDistributions change", + "passed": true, + "evidence": "Report attests 'UserStatusDistributions.cs (XML doc summary) \u2014 verified current: the `` was rewritten in the same edit and now reads 80% Confirmed, 8% Invited, 7% Accepted, 5% Revoked, matching the new tuples exactly'." + }, + { + "text": "Ends with a per-scope report", + "passed": true, + "evidence": "The 'Closing summary' section enumerates each scope (both changed files' XML docs, Data/README.md, Seeder/README.md, CLAUDE.md, root README) with per-scope status/verdict." + } + ], + "notes": [] + }, + { + "eval_id": 4, + "configuration": "with_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Names what the doc says (the deterministic claim) and what the code does (wall-clock non-determinism) as a concrete disagreement", + "passed": true, + "evidence": "Report quotes the invariant claim ('Deterministic. Seeded randomness means same org ID \u2192 same test data \u2192 reproducible debugging') and calls it 'contradicted by `UsernameDistributions.Current` reading wall-clock state'." + }, + { + "text": "Either edits util/Seeder/Data/README.md's determinism claim, or surfaces the contradiction with a specific recommendation \u2014 does not silently defer", + "passed": true, + "evidence": "Report says 'util/Seeder/Data/README.md \u2014 updated. ... Rewritten to \"Deterministic by default\" and to name the specific opt-out distribution.' Similar updates to Seeder/README.md and CLAUDE.md are also described." + }, + { + "text": "Still completes per-scope verification for the actual UserStatusDistributions change", + "passed": true, + "evidence": "Report attests 'UserStatusDistributions.cs (XML doc) \u2014 verified current. The doc summary was updated in the same diff to match the new 80/8/7/5 split'." + }, + { + "text": "Ends with a per-scope report", + "passed": true, + "evidence": "The 'Documentation parity' section provides a per-scope roll-up of every enumerated scope (both source XML docs, Data/README.md, Seeder/README.md, CLAUDE.md, Seeds docs, root README)." + } + ], + "notes": [] + }, + { + "eval_id": 4, + "configuration": "with_skill", + "run_number": 4, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Names what the doc says (the deterministic claim) and what the code does (wall-clock non-determinism) as a concrete disagreement", + "passed": true, + "evidence": "Report quotes the 'Design Principles' bullet ('**Deterministic.** Seeded randomness means same org ID -> same test data -> reproducible debugging.') and states `UsernameDistributions.Current` 'publicly exposes a distribution for which \"same org ID -> same test data\" no longer holds'." + }, + { + "text": "Either edits util/Seeder/Data/README.md's determinism claim, or surfaces the contradiction with a specific recommendation \u2014 does not silently defer", + "passed": true, + "evidence": "Edited: 'util/Seeder/Data/README.md -- Design Principles \"Deterministic\" bullet.' Softened to '**Deterministic by default.** ... Time-sampling selectors such as `UsernameDistributions.Current` opt out and flag themselves on the type.'" + }, + { + "text": "Still completes per-scope verification for the actual UserStatusDistributions change", + "passed": true, + "evidence": "Report attests 'UserStatusDistributions.cs XML docs -- verified current ... The `///` summary was updated in the same diff (85/5/5/5 -> 80/8/7/5) to match the new tuple values. No drift.'" + }, + { + "text": "Ends with a per-scope report", + "passed": true, + "evidence": "Closing summary provides a bulleted per-scope roll-up: 'util/Seeder/Data (README.md) -- updated ...', 'util/Seeder (README.md) -- updated ...', 'CLAUDE.md -- verified current ...', 'Repo root README.md -- verified current ...', plus XML doc surfaces." + } + ], + "notes": [] + }, + { + "eval_id": 4, + "configuration": "with_skill", + "run_number": 5, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Names what the doc says (the deterministic claim) and what the code does (wall-clock non-determinism) as a concrete disagreement", + "passed": true, + "evidence": "Report quotes 'Deterministic. Seeded randomness means same org ID \u2192 same test data \u2192 reproducible debugging' and states 'Adding a public non-deterministic distribution to the Data system contradicts the \"Deterministic\" architectural bullet at this altitude.'" + }, + { + "text": "Either edits util/Seeder/Data/README.md's determinism claim, or surfaces the contradiction with a specific recommendation \u2014 does not silently defer", + "passed": true, + "evidence": "Did not silently defer \u2014 explicitly surfaced with a specific pathway: 'Please confirm what to do with the `Current` property \u2014 if it stays, I will update this bullet to carve out the exception.' Multiple design options (weaken the principle, remove the property, gate it) are named." + }, + { + "text": "Still completes per-scope verification for the actual UserStatusDistributions change", + "passed": true, + "evidence": "Report attests 'UserStatusDistributions.cs XML doc surface \u2014 verified current' with the specific 85/5/5/5 \u2192 80/8/7/5 comparison quoted." + }, + { + "text": "Ends with a per-scope report", + "passed": true, + "evidence": "The 'Documentation parity (per-scope roll-up)' section enumerates every scope: both files' XML docs, Data/README.md, Seeder/README.md, CLAUDE.md, and repo root, each with an explicit verdict." + } + ], + "notes": [] + }, + { + "eval_id": 5, + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Removes the policy-enterprise entry from util/Seeder/Seeds/docs/presets.md's preset catalog", + "passed": true, + "evidence": "Diff removes the `policy-enterprise | All policies except requireSso and require2fa | pinnacle-designs | starter-team | \u2014` row and the follow-up 'has no ciphers' sentence from presets.md." + }, + { + "text": "Checks util/Seeder/Seeds/README.md and util/Seeder/Seeds/docs/scenarios/ for inbound references to the deleted preset and updates any that reference it", + "passed": true, + "evidence": "Report states 'util/Seeder/Seeds/README.md \u2014 verified current' and 'util/Seeder/Seeds/docs/scenarios/*.md \u2014 verified current. Grep against the whole scenarios/ tree found zero mentions of policy-enterprise or pinnacle.'" + }, + { + "text": "Verifies the higher scopes (util/Seeder, repo root) and reports per scope", + "passed": true, + "evidence": "Report explicitly attests 'util/Seeder/README.md \u2014 verified current', 'util/Seeder/CLAUDE.md \u2014 verified current', and 'Repo root README.md \u2014 verified current'." + }, + { + "text": "Does not leave catalog rows or links pointing at the removed preset", + "passed": true, + "evidence": "Repo-wide grep confirmation in the report ('grep confirmed no reference to policy-enterprise or pinnacle') and diff shows both catalog row and follow-up sentence removed." + } + ], + "notes": [] + }, + { + "eval_id": 5, + "configuration": "with_skill", + "run_number": 2, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Removes the policy-enterprise entry from util/Seeder/Seeds/docs/presets.md's preset catalog", + "passed": true, + "evidence": "Diff shows the `policy-enterprise` row and the follow-up 'no ciphers' sentence removed from the Features table in presets.md." + }, + { + "text": "Checks util/Seeder/Seeds/README.md and util/Seeder/Seeds/docs/scenarios/ for inbound references to the deleted preset and updates any that reference it", + "passed": true, + "evidence": "Report attests 'util/Seeder/Seeds/README.md \u2014 catalog is folder-level' and 'util/Seeder/Seeds/docs/scenarios/*.md \u2014 verified current: no scenario references features.policy-enterprise' (checked via grep)." + }, + { + "text": "Verifies the higher scopes (util/Seeder, repo root) and reports per scope", + "passed": true, + "evidence": "Report attests 'util/Seeder/README.md \u2014 verified current', 'util/Seeder/CLAUDE.md \u2014 verified current', and 'repo root (README.md, CONTRIBUTING.md) \u2014 verified current: no Seeder-preset references'." + }, + { + "text": "Does not leave catalog rows or links pointing at the removed preset", + "passed": true, + "evidence": "Diff removes both the row and the follow-up sentence; report notes only Seeds/docs/presets.md matched grep, and all references are cleared." + } + ], + "notes": [] + }, + { + "eval_id": 5, + "configuration": "with_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Removes the policy-enterprise entry from util/Seeder/Seeds/docs/presets.md's preset catalog", + "passed": true, + "evidence": "Diff removes the policy-enterprise catalog row and its follow-up 'has no ciphers' sentence; report says 'Updated: removed the row and the trailing sentence'." + }, + { + "text": "Checks util/Seeder/Seeds/README.md and util/Seeder/Seeds/docs/scenarios/ for inbound references to the deleted preset and updates any that reference it", + "passed": true, + "evidence": "Report says 'util/Seeder/Seeds/README.md - verified current: it lists preset folders' and 'util/Seeder/Seeds/docs/scenarios/README.md (and sibling scenario files) - verified current: scenarios map problems to preset/organization commands and none name features.policy-enterprise'." + }, + { + "text": "Verifies the higher scopes (util/Seeder, repo root) and reports per scope", + "passed": true, + "evidence": "Report attests 'util/Seeder/README.md - verified current', 'util/Seeder/CLAUDE.md - verified current', and 'repo root README.md / CONTRIBUTING.md / SECURITY.md ... - verified current'." + }, + { + "text": "Does not leave catalog rows or links pointing at the removed preset", + "passed": true, + "evidence": "Report states 'grep -rn -iE \"policy-enterprise|pinnacle\" .' returns no hits after edit, confirming no dangling references." + } + ], + "notes": [] + }, + { + "eval_id": 5, + "configuration": "with_skill", + "run_number": 4, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Removes the policy-enterprise entry from util/Seeder/Seeds/docs/presets.md's preset catalog", + "passed": true, + "evidence": "Diff removes the `policy-enterprise` row from the Features table and the trailing 'no ciphers' sentence; report confirms the edit." + }, + { + "text": "Checks util/Seeder/Seeds/README.md and util/Seeder/Seeds/docs/scenarios/ for inbound references to the deleted preset and updates any that reference it", + "passed": true, + "evidence": "Report says 'util/Seeder/Seeds/README.md \u2014 high-level Seeds overview; describes only the folder layout... Unaffected' and 'util/Seeder/Seeds/docs/scenarios/permission-testing.md (and other scenarios) \u2014 grep confirmed no reference to policy-enterprise or pinnacle. Unaffected'." + }, + { + "text": "Verifies the higher scopes (util/Seeder, repo root) and reports per scope", + "passed": true, + "evidence": "Report attests 'util/Seeder/ (README.md, CLAUDE.md) \u2014 verified current' and 'repo root (README.md, CONTRIBUTING.md) \u2014 verified current: repo-wide grep for policy-enterprise and pinnacle finds no hits'." + }, + { + "text": "Does not leave catalog rows or links pointing at the removed preset", + "passed": true, + "evidence": "Report states 'Repo-wide grep for policy-enterprise and pinnacle across *.md, *.cs, *.json returned hits only in util/Seeder/Seeds/docs/presets.md (the two lines edited)', and diff shows those removed." + } + ], + "notes": [] + }, + { + "eval_id": 5, + "configuration": "with_skill", + "run_number": 5, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Removes the policy-enterprise entry from util/Seeder/Seeds/docs/presets.md's preset catalog", + "passed": true, + "evidence": "Diff removes the policy-enterprise row and the 'no ciphers' follow-up sentence; report says 'deleted the policy-enterprise row from the Features table and its follow-up no ciphers explanatory sentence'." + }, + { + "text": "Checks util/Seeder/Seeds/README.md and util/Seeder/Seeds/docs/scenarios/ for inbound references to the deleted preset and updates any that reference it", + "passed": true, + "evidence": "Report attests 'Seeds/README.md: verified current' and 'Seeds/docs/scenarios/README.md and the seven scenario files: verified current. The scenarios table maps problems to qa.*, scale.*, and individual.* presets; no scenario pointed at features.policy-enterprise'." + }, + { + "text": "Verifies the higher scopes (util/Seeder, repo root) and reports per scope", + "passed": true, + "evidence": "Report attests 'util/Seeder/README.md \u2014 verified current' and 'Repo root README.md \u2014 verified current: Root-level content is deploy/build/install oriented'." + }, + { + "text": "Does not leave catalog rows or links pointing at the removed preset", + "passed": true, + "evidence": "Report states grep for 'policy-enterprise' and 'pinnacle' showed 'only prose hits were in util/Seeder/Seeds/docs/presets.md', and diff removes both hits." + } + ], + "notes": [] + }, + { + "eval_id": 6, + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 5, + "failed": 0, + "total": 5, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and identifies that its `attachmentVersion` axis table enumerates `v0 | v1 | v2` and the cipher/attachment invariant needs revisiting for `v3`", + "passed": true, + "evidence": "Report says README.md 'previously enumerated `v0/v1/v2` as the axis and asserted a two-invariant rule that only covered v0/v1/v2' and identifies both drifts before editing." + }, + { + "text": "Updates util/Seeder/README.md's attachmentVersion enumeration and invariant rather than only attesting", + "passed": true, + "evidence": "Diff extends the Invariant line with 'A `v3` (org-key-wrapped) attachment additionally requires an org-owned host cipher; personal-vault ciphers cannot host `v3`' and inserts a paragraph describing v3 in the enum/factory/resolver, plus updates the callout to `v0/v1/v2/v3`." + }, + { + "text": "Enumerates the changed source files (Pipeline/, Factories/) as below-component source-embedded surfaces in Step 2, distinct from the tree-level README scopes", + "passed": true, + "evidence": "'Scopes enumerated' section lists AttachmentSchemeType.cs, AttachmentSeeder.cs, and AttachmentSchemeResolver.cs each explicitly as '(source-embedded XML doc surface)', separate from util/Seeder/ and repo-root README entries." + }, + { + "text": "Also checks util/Seeder/Seeds/ documentation for references to the attachmentVersion key", + "passed": true, + "evidence": "Report explicitly inspects `Seeds/schemas/cipher.schema.json`, `Seeds/docs/presets.md`, `Seeds/docs/scenarios/encryption-modes.md`, `encryption-modes-org.md`, and `Seeds/README.md` and attests each with a grounded reason." + }, + { + "text": "Does not treat the already-updated XML doc comments as sufficient for the higher-scope READMEs", + "passed": true, + "evidence": "Even though XML docs on the three source files are marked verified current, README.md and CLAUDE.md are explicitly updated with v3 content in the diff (both files show substantive additions)." + } + ], + "notes": [] + }, + { + "eval_id": 6, + "configuration": "with_skill", + "run_number": 2, + "result": { + "pass_rate": 0.8, + "passed": 4, + "failed": 1, + "total": 5, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and identifies that its `attachmentVersion` axis table enumerates `v0 | v1 | v2` and the cipher/attachment invariant needs revisiting for `v3`", + "passed": true, + "evidence": "Report states 'the \"Encryption Schemes\" table row for `attachmentVersion` still listed only `v0 \u00b7 v1 \u00b7 v2`, and the following invariant sentence mentioned only the v2/cipherKey pairing.'" + }, + { + "text": "Updates util/Seeder/README.md's attachmentVersion enumeration and invariant rather than only attesting", + "passed": true, + "evidence": "Diff changes the table Values column from `v0 \u00b7 v1 \u00b7 v2` to `v0 \u00b7 v1 \u00b7 v2 \u00b7 v3` and updates the Invariant line to include 'a `v3` attachment requires an org-owned cipher (personal-vault ciphers cannot host `v3`).'" + }, + { + "text": "Enumerates the changed source files (Pipeline/, Factories/) as below-component source-embedded surfaces in Step 2, distinct from the tree-level README scopes", + "passed": true, + "evidence": "Union list explicitly separates items 1-3 (AttachmentSchemeType.cs, AttachmentSeeder.cs, AttachmentSchemeResolver.cs) as '(XML doc surface)' from items 4-6 (Seeder README, CLAUDE.md, repo root README)." + }, + { + "text": "Also checks util/Seeder/Seeds/ documentation for references to the attachmentVersion key", + "passed": false, + "evidence": "The report only mentions `Seeds/schemas/cipher.schema.json` under 'Notes outside the doc-parity remit'; it does not read or attest util/Seeder/Seeds/README.md, Seeds/docs/presets.md, or Seeds/docs/scenarios/encryption-modes*.md for attachmentVersion references." + }, + { + "text": "Does not treat the already-updated XML doc comments as sufficient for the higher-scope READMEs", + "passed": true, + "evidence": "Diff makes substantive edits to both util/Seeder/README.md (table row + invariant) and util/Seeder/CLAUDE.md (bullet, invariant, and wire mapping), rather than resting on the class-level XML updates." + } + ], + "notes": [] + }, + { + "eval_id": 6, + "configuration": "with_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 5, + "failed": 0, + "total": 5, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and identifies that its `attachmentVersion` axis table enumerates `v0 | v1 | v2` and the cipher/attachment invariant needs revisiting for `v3`", + "passed": true, + "evidence": "Report says 'the \"Encryption Schemes\" table row for `attachmentVersion` was extended with `v3`' and 'the invariant sentence now names v3's org-owned-host requirement' \u2014 showing prior state was v0/v1/v2 without v3." + }, + { + "text": "Updates util/Seeder/README.md's attachmentVersion enumeration and invariant rather than only attesting", + "passed": true, + "evidence": "Diff changes the table Values column to `v0 \u00b7 v1 \u00b7 v2 \u00b7 v3` and extends the Invariant with 'a `v3` attachment requires an org-owned host cipher (personal-vault ciphers cannot host v3)', plus updates the V1/V2 callout to `v0/v1/v2/v3`." + }, + { + "text": "Enumerates the changed source files (Pipeline/, Factories/) as below-component source-embedded surfaces in Step 2, distinct from the tree-level README scopes", + "passed": true, + "evidence": "'Scopes and surfaces enumerated' explicitly lists 'Source-embedded XML docs on the three changed files' before enumerating the tree-level scopes (util/Seeder, util, repo root)." + }, + { + "text": "Also checks util/Seeder/Seeds/ documentation for references to the attachmentVersion key", + "passed": true, + "evidence": "Report explicitly discusses 'Sibling documentation under `util/Seeder/Seeds/docs/` (scenarios and presets)' and validates it against `Seeds/schemas/cipher.schema.json` and `Steps/CreateCipherAttachmentsStep.cs`, concluding v3 is not fixture-reachable so those pages stay accurate." + }, + { + "text": "Does not treat the already-updated XML doc comments as sufficient for the higher-scope READMEs", + "passed": true, + "evidence": "Diff makes substantive multi-line edits to both util/Seeder/README.md (table row, invariant, callout) and util/Seeder/CLAUDE.md (bullet, invariant, wire mapping, callout) rather than resting on the class-level XML doc updates." + } + ], + "notes": [] + }, + { + "eval_id": 6, + "configuration": "with_skill", + "run_number": 4, + "result": { + "pass_rate": 0.8, + "passed": 4, + "failed": 1, + "total": 5, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and identifies that its `attachmentVersion` axis table enumerates `v0 | v1 | v2` and the cipher/attachment invariant needs revisiting for `v3`", + "passed": true, + "evidence": "'Docs read' lists 'util/Seeder/README.md \u2014 the \"Encryption Schemes\" section frames both axes as fixture-selectable and its \"Values\" column enumerates only fixture-accepted values.'" + }, + { + "text": "Updates util/Seeder/README.md's attachmentVersion enumeration and invariant rather than only attesting", + "passed": false, + "evidence": "The diff does not touch util/Seeder/README.md at all; the report marks it 'verified current' arguing 'the \"Values\" column is scoped to fixture-selectable values' \u2014 this is attesting rather than updating." + }, + { + "text": "Enumerates the changed source files (Pipeline/, Factories/) as below-component source-embedded surfaces in Step 2, distinct from the tree-level README scopes", + "passed": true, + "evidence": "'Source-embedded doc surfaces on changed files' lists the three changed .cs files as XML doc surfaces, separately from 'Documented ancestor scopes' (Enums/, Factories/, Pipeline/, util/Seeder/, util/, repo root)." + }, + { + "text": "Also checks util/Seeder/Seeds/ documentation for references to the attachmentVersion key", + "passed": true, + "evidence": "'Docs read' lists Seeds/README.md, Seeds/docs/architecture.md, Seeds/docs/fixtures.md, Seeds/docs/presets.md (line 103), Seeds/docs/scenarios/encryption-modes.md, encryption-modes-org.md, and Seeds/schemas/cipher.schema.json with per-file findings on attachmentVersion content." + }, + { + "text": "Does not treat the already-updated XML doc comments as sufficient for the higher-scope READMEs", + "passed": true, + "evidence": "Diff makes a substantive multi-paragraph edit to util/Seeder/CLAUDE.md (bullet, invariant, wire mapping) rather than resting on the XML doc updates. (Note: README.md was left unchanged, but the report's reasoning is fixture-scope, not 'XML docs suffice'.)" + } + ], + "notes": [] + }, + { + "eval_id": 6, + "configuration": "with_skill", + "run_number": 5, + "result": { + "pass_rate": 1.0, + "passed": 5, + "failed": 0, + "total": 5, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and identifies that its `attachmentVersion` axis table enumerates `v0 | v1 | v2` and the cipher/attachment invariant needs revisiting for `v3`", + "passed": true, + "evidence": "Report reads util/Seeder/README.md and edits it 'after the axis table' plus 'Extended the Invariant line with the v3 constraint (org-owned cipher + wrapped org key)', showing recognition that both drifted from v0/v1/v2." + }, + { + "text": "Updates util/Seeder/README.md's attachmentVersion enumeration and invariant rather than only attesting", + "passed": true, + "evidence": "Diff adds a full v3 paragraph after the table, updates the Invariant sentence to add 'A `v3` attachment additionally requires an org-owned cipher ... and a wrapped org key', and changes the callout to `v0/v1/v2/v3`." + }, + { + "text": "Enumerates the changed source files (Pipeline/, Factories/) as below-component source-embedded surfaces in Step 2, distinct from the tree-level README scopes", + "passed": true, + "evidence": "'Scopes and surfaces enumerated' lists AttachmentSchemeType.cs, AttachmentSeeder.cs, and AttachmentSchemeResolver.cs each as '\u2014 source-embedded XML doc (touched/new)', then separately lists util/Seeder/README.md, CLAUDE.md, Seeds/, and repo root." + }, + { + "text": "Also checks util/Seeder/Seeds/ documentation for references to the attachmentVersion key", + "passed": true, + "evidence": "Report reads Seeds/README.md, Seeds/docs/{architecture,fixtures,verification}.md, Seeds/docs/presets.md, and Seeds/docs/scenarios/{encryption-modes,encryption-modes-org,README}.md, then edits presets.md and encryption-modes-org.md to drop the stale 'all three attachment schemes (v0/v1/v2)' completeness claim." + }, + { + "text": "Does not treat the already-updated XML doc comments as sufficient for the higher-scope READMEs", + "passed": true, + "evidence": "Diff makes substantive multi-file edits to util/Seeder/README.md, util/Seeder/CLAUDE.md, Seeds/docs/presets.md, and Seeds/docs/scenarios/encryption-modes-org.md, on top of the class-level XML doc changes." + } + ], + "notes": [] + }, + { + "eval_id": 7, + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 3, + "failed": 0, + "total": 3, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Treats util/Seeder/CLAUDE.md as documentation in scope and updates its `dotnet test` command(s) to the new path", + "passed": true, + "evidence": "Report says 'util/Seeder/CLAUDE.md \u2014 updated. Three references... All three now point at test/Seeder.IntegrationTest/.' Diff confirms updates to both `dotnet test test/Seeder.IntegrationTest/` commands and the Steps/ prose reference." + }, + { + "text": "Checks the other Seeder READMEs for references to the old test project path and updates any found", + "passed": true, + "evidence": "Report explicitly states 'util/Seeder/README.md \u2014 verified current. The README describes the Seeder library's architecture... does not reference the integration test project by name.'" + }, + { + "text": "Reports per scope, marking each scope as either updated or verified current", + "passed": true, + "evidence": "Report contains 'Documentation parity' section listing each scope (repo root, test/, util/Seeder README, util/Seeder CLAUDE.md, bump-rust-sdk) as either 'updated' or 'verified current'." + } + ], + "notes": [] + }, + { + "eval_id": 7, + "configuration": "with_skill", + "run_number": 2, + "result": { + "pass_rate": 1.0, + "passed": 3, + "failed": 0, + "total": 3, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Treats util/Seeder/CLAUDE.md as documentation in scope and updates its `dotnet test` command(s) to the new path", + "passed": true, + "evidence": "Report says 'util/Seeder/CLAUDE.md \u2014 updated. Contained three drifted references... All three now point at test/Seeder.IntegrationTest/'. Diff confirms `dotnet test test/Seeder.IntegrationTest/` replaces both old commands." + }, + { + "text": "Checks the other Seeder READMEs for references to the old test project path and updates any found", + "passed": true, + "evidence": "Report states 'util/Seeder/README.md \u2014 verified current. The README describes the seeder library's patterns... It does not name any test project.' Also checks test/INTEGRATION_TEST.md." + }, + { + "text": "Reports per scope, marking each scope as either updated or verified current", + "passed": true, + "evidence": "Report ends with 'Documentation parity:' text block itemizing every scope (repo root, util/Seeder README, util/Seeder CLAUDE.md, test/, bump-rust-sdk) as either 'updated' or 'verified current'." + } + ], + "notes": [] + }, + { + "eval_id": 7, + "configuration": "with_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 3, + "failed": 0, + "total": 3, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Treats util/Seeder/CLAUDE.md as documentation in scope and updates its `dotnet test` command(s) to the new path", + "passed": true, + "evidence": "Report says 'util/Seeder/CLAUDE.md \u2014 updated' with three concrete references retargeted. Diff shows `dotnet test test/Seeder.IntegrationTest/` replacing the old path in both command blocks." + }, + { + "text": "Checks the other Seeder READMEs for references to the old test project path and updates any found", + "passed": true, + "evidence": "Report states 'util/Seeder/README.md \u2014 verified current. Describes patterns... Its only SeederApi reference is to the SeederApi application, not the integration-test project. No stale path references.'" + }, + { + "text": "Reports per scope, marking each scope as either updated or verified current", + "passed": true, + "evidence": "Report has 'Per-scope attestation' section under 'Documentation parity:' listing each scope (repo root, util/Seeder README, util/Seeder CLAUDE.md, test/, bump-rust-sdk) as either 'updated' or 'verified current'." + } + ], + "notes": [] + }, + { + "eval_id": 7, + "configuration": "with_skill", + "run_number": 4, + "result": { + "pass_rate": 1.0, + "passed": 3, + "failed": 0, + "total": 3, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Treats util/Seeder/CLAUDE.md as documentation in scope and updates its `dotnet test` command(s) to the new path", + "passed": true, + "evidence": "Report says 'util/Seeder/CLAUDE.md \u2014 updated: three test/SeederApi.IntegrationTest/ references... retargeted to test/Seeder.IntegrationTest/'. Diff shows both dotnet test commands updated to the new path." + }, + { + "text": "Checks the other Seeder READMEs for references to the old test project path and updates any found", + "passed": true, + "evidence": "Report says 'util/Seeder/README.md \u2014 read. No references to any *.IntegrationTest project or to test file paths. Nothing at this scope drifted.'" + }, + { + "text": "Reports per scope, marking each scope as either updated or verified current", + "passed": true, + "evidence": "Report has 'Per-scope attestation' section under 'Documentation parity:' with each scope (util/Seeder README, util/Seeder CLAUDE.md, test/, repo root) marked as either 'updated' or 'verified current'." + } + ], + "notes": [] + }, + { + "eval_id": 7, + "configuration": "with_skill", + "run_number": 5, + "result": { + "pass_rate": 1.0, + "passed": 3, + "failed": 0, + "total": 3, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Treats util/Seeder/CLAUDE.md as documentation in scope and updates its `dotnet test` command(s) to the new path", + "passed": true, + "evidence": "Report lists 'util/Seeder/CLAUDE.md \u2014 Updated three references... Command example: dotnet test test/SeederApi.IntegrationTest/ -> dotnet test test/Seeder.IntegrationTest/'. Diff confirms the updates." + }, + { + "text": "Checks the other Seeder READMEs for references to the old test project path and updates any found", + "passed": true, + "evidence": "Report explicitly lists reading util/Seeder/README.md, util/SeederApi/README.md, util/SeederUtility/README.md, util/Seeder/Data/README.md, util/Seeder/Seeds/README.md, all marked 'verified current' with grounded reasons." + }, + { + "text": "Reports per scope, marking each scope as either updated or verified current", + "passed": true, + "evidence": "Report has 'Summary' section titled 'Documentation parity:' listing every scope (repo root, test/, util/Seeder/, util/Seeder/Data, util/SeederApi, util/SeederUtility, bump-rust-sdk) as either 'updated' or 'verified current'." + } + ], + "notes": [] + }, + { + "eval_id": 8, + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Recognizes the diff as doc-comment-only inside a source file misclassified as code by the Stop hook", + "passed": true, + "evidence": "Report states 'This diff is precisely the third case named in that rule -- a doc-comment-only edit (XML `` tweaks) that the Stop hook classified as a code change because it touched a `.cs` file.'" + }, + { + "text": "Dismisses the check with a one-line reason naming the false-positive class", + "passed": true, + "evidence": "Closing summary: 'Documentation parity: dismissed -- doc-comment-only edit (XML `` typo fix + line reflow in `util/Seeder/Data/GeneratorContext.cs`), no documentation obligation.'" + }, + { + "text": "Does not walk the per-scope verification tree after dismissing", + "passed": true, + "evidence": "'Scopes/surfaces enumerated: None. Step 1 short-circuited the workflow before scope enumeration (Step 2)' and 'Docs read: None beyond the file itself'." + }, + { + "text": "Does not make any additional documentation edits", + "passed": true, + "evidence": "worktree.diff shows only the original two doc-comment hunks in GeneratorContext.cs, no additional edits. Report states 'Docs edited: None.'" + } + ], + "notes": [] + }, + { + "eval_id": 8, + "configuration": "with_skill", + "run_number": 2, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Recognizes the diff as doc-comment-only inside a source file misclassified as code by the Stop hook", + "passed": true, + "evidence": "Report: 'The doc-parity Stop hook flags edits inside `.cs` source files as code, but this diff is a **doc-comment-only edit** \u2014 one of the three named false-positive classes.'" + }, + { + "text": "Dismisses the check with a one-line reason naming the false-positive class", + "passed": true, + "evidence": "Closing summary: 'Documentation parity: dismissed \u2014 doc-comment-only edit inside `util/Seeder/Data/GeneratorContext.cs`, no documentation obligation triggered.'" + }, + { + "text": "Does not walk the per-scope verification tree after dismissing", + "passed": true, + "evidence": "'Scopes enumerated: none \u2014 dismissed before Step 2. Docs read: none \u2014 the SKILL.md itself, plus the diff. No READMEs, module docs, or other documentation surfaces were opened.'" + }, + { + "text": "Does not make any additional documentation edits", + "passed": true, + "evidence": "worktree.diff shows only original two hunks in GeneratorContext.cs; report explicitly says 'Edits: none in the worktree beyond the pre-existing staged change'." + } + ], + "notes": [] + }, + { + "eval_id": 8, + "configuration": "with_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Recognizes the diff as doc-comment-only inside a source file misclassified as code by the Stop hook", + "passed": true, + "evidence": "Report: 'This diff matches that class exactly: Both hunks live in `///` XML doc comment blocks... calling out \"a doc-comment-only edit that the Stop hook classified as code\" as a dismissal class.'" + }, + { + "text": "Dismisses the check with a one-line reason naming the false-positive class", + "passed": true, + "evidence": "Closing summary: 'Documentation parity: dismissed \u2014 doc-comment-only diff (typo fix and line re-flow inside `///` XML doc comments on `GeneratorContext` and `DeriveStableSeed`), no documentation obligation.'" + }, + { + "text": "Does not walk the per-scope verification tree after dismissing", + "passed": true, + "evidence": "'Enumeration was short-circuited by the tripwire dismissal per the skill. No READMEs, `docs/` directories, diagrams, or other source-embedded doc surfaces were read or edited.'" + }, + { + "text": "Does not make any additional documentation edits", + "passed": true, + "evidence": "worktree.diff shows only original two hunks; report says 'Files edited: None.'" + } + ], + "notes": [] + }, + { + "eval_id": 8, + "configuration": "with_skill", + "run_number": 4, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Recognizes the diff as doc-comment-only inside a source file misclassified as code by the Stop hook", + "passed": true, + "evidence": "Report: 'Every added and removed line is inside an XML doc comment; the doc-parity Stop hook fired because it treats `.cs` files as code and cannot see that the modified lines are all `///` comment text.'" + }, + { + "text": "Dismisses the check with a one-line reason naming the false-positive class", + "passed": true, + "evidence": "Closing summary: 'Documentation parity: dismissed - doc-comment-only edit (typo fix and line re-flow inside XML `` blocks in `util/Seeder/Data/GeneratorContext.cs`); no documentation obligation.'" + }, + { + "text": "Does not walk the per-scope verification tree after dismissing", + "passed": true, + "evidence": "'Steps 2-5 (scope enumeration, per-scope judgment, per-scope attestation) do not run. No documented ancestor scopes were enumerated, no READMEs were read.'" + }, + { + "text": "Does not make any additional documentation edits", + "passed": true, + "evidence": "worktree.diff shows only original two hunks; report: 'no other docs were edited... The diff on disk remains exactly the user's original edit.'" + } + ], + "notes": [] + }, + { + "eval_id": 8, + "configuration": "with_skill", + "run_number": 5, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Recognizes the diff as doc-comment-only inside a source file misclassified as code by the Stop hook", + "passed": true, + "evidence": "Report: 'This diff falls squarely in the third class: every changed line is a `///` XML doc comment. The hook fired because the file `.cs` extension makes it look like code.'" + }, + { + "text": "Dismisses the check with a one-line reason naming the false-positive class", + "passed": true, + "evidence": "Closing summary: 'Documentation parity: dismissed \u2014 doc-comment-only edit (XML `` typo fix and line re-flow in `util/Seeder/Data/GeneratorContext.cs`), no documentation obligation triggered.'" + }, + { + "text": "Does not walk the per-scope verification tree after dismissing", + "passed": true, + "evidence": "'Scopes enumerated: None. Step 2 is skipped by the Step 1 dismissal rule.' and 'Docs read: None beyond the changed XML doc comments themselves.'" + }, + { + "text": "Does not make any additional documentation edits", + "passed": true, + "evidence": "worktree.diff shows only original two hunks; report says 'Docs edited: None.'" + } + ], + "notes": [] + }, + { + "eval_id": 9, + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Identifies util/Seeder/CrossHarnessBridge as the artifact the design doc describes, and test/Common/Helpers/EncryptionHarness as a dependency the bridge imports rather than a subject of the doc", + "passed": true, + "evidence": "'The design doc describes the bridge \u2014 its responsibility, its lifecycle inside the Seeder pipeline, and the contract it imposes on the harness. The scope that owns the bridge is util/Seeder/CrossHarnessBridge/.' Also: 'Do not put it under test/Common/Helpers/EncryptionHarness/. The harness's XML doc already names the bridge as the consumer...'" + }, + { + "text": "Places the design doc at util/Seeder/CrossHarnessBridge/ (README.md, design.md, or a docs/ page inside that directory) \u2014 co-located with the concept it describes", + "passed": true, + "evidence": "'Primary landing spot: util/Seeder/CrossHarnessBridge/README.md. If the design is long enough to warrant a doc set... promote it to util/Seeder/CrossHarnessBridge/docs/design.md'" + }, + { + "text": "Cites the documentation standard's placement or decision rule as the reason, applying 'what the doc describes' as the placement input rather than 'what the code imports'", + "passed": true, + "evidence": "'Under the documentation standard's co-location rule, docs live at the scope that owns the behavior they describe. The design doc describes the bridge...'" + }, + { + "text": "Does not treat cross-container imports as evidence that the doc should live at those containers' shared ancestor (e.g., repo root or a shared docs/ folder)", + "passed": true, + "evidence": "Places doc at util/Seeder/CrossHarnessBridge/ (not at a shared ancestor). Explicitly rejects test-side placement: 'documenting the whole bridge on the helper's side inverts ownership.'" + } + ], + "notes": [] + }, + { + "eval_id": 9, + "configuration": "with_skill", + "run_number": 2, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Identifies util/Seeder/CrossHarnessBridge as the artifact the design doc describes, and test/Common/Helpers/EncryptionHarness as a dependency the bridge imports rather than a subject of the doc", + "passed": true, + "evidence": "'The bridge is a Seeder concept, not a test-side concept... test/Common/Helpers/EncryptionHarness/ is a passive dependency - it exposes key material; it does not know about the bridge. Documenting the bridge next to its callers, not next to its dependencies...'" + }, + { + "text": "Places the design doc at util/Seeder/CrossHarnessBridge/ (README.md, design.md, or a docs/ page inside that directory) \u2014 co-located with the concept it describes", + "passed": true, + "evidence": "'Recommendation: util/Seeder/CrossHarnessBridge/README.md'; closing: 'The design doc for the new bridge should live at util/Seeder/CrossHarnessBridge/README.md.'" + }, + { + "text": "Cites the documentation standard's placement or decision rule as the reason, applying 'what the doc describes' as the placement input rather than 'what the code imports'", + "passed": true, + "evidence": "'Placement rule. The Bitwarden documentation standard says a doc lives at the lowest scope that fully covers what it describes. The design doc is about how this bridge module works...'" + }, + { + "text": "Does not treat cross-container imports as evidence that the doc should live at those containers' shared ancestor (e.g., repo root or a shared docs/ folder)", + "passed": true, + "evidence": "'Nothing higher fits.' Placement is inside util/Seeder/CrossHarnessBridge/, not at any shared ancestor of util/ and test/." + } + ], + "notes": [] + }, + { + "eval_id": 9, + "configuration": "with_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Identifies util/Seeder/CrossHarnessBridge as the artifact the design doc describes, and test/Common/Helpers/EncryptionHarness as a dependency the bridge imports rather than a subject of the doc", + "passed": true, + "evidence": "'The bridge module is a Seeder concern that consumes a test-side helper as a dependency \u2014 the primary owner is util/Seeder/CrossHarnessBridge/, not test/Common/Helpers/EncryptionHarness/.' Also: 'The harness is a dependency of the bridge; the bridge's design doesn't describe the harness's internals...'" + }, + { + "text": "Places the design doc at util/Seeder/CrossHarnessBridge/ (README.md, design.md, or a docs/ page inside that directory) \u2014 co-located with the concept it describes", + "passed": true, + "evidence": "'Recommended: util/Seeder/CrossHarnessBridge/README.md.' Also: 'fold it out to util/Seeder/CrossHarnessBridge/docs/'" + }, + { + "text": "Cites the documentation standard's placement or decision rule as the reason, applying 'what the doc describes' as the placement input rather than 'what the code imports'", + "passed": true, + "evidence": "'The documentation standard's placement rule is: docs live at the scope they describe.'" + }, + { + "text": "Does not treat cross-container imports as evidence that the doc should live at those containers' shared ancestor (e.g., repo root or a shared docs/ folder)", + "passed": true, + "evidence": "'Not under test/Common/Helpers/EncryptionHarness/. The harness is a dependency of the bridge... putting the bridge doc on the test side inverts the ownership direction.' Places doc inside util/Seeder/CrossHarnessBridge/, not at a shared ancestor." + } + ], + "notes": [] + }, + { + "eval_id": 9, + "configuration": "with_skill", + "run_number": 4, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Identifies util/Seeder/CrossHarnessBridge as the artifact the design doc describes, and test/Common/Helpers/EncryptionHarness as a dependency the bridge imports rather than a subject of the doc", + "passed": true, + "evidence": "'The bridge is a Seeder-owned subsystem; the class file is at util/Seeder/CrossHarnessBridge/CrossHarnessBridge.cs.' Also: 'The test/Common/Helpers/EncryptionHarness/ side is a helper, not a subsystem.'" + }, + { + "text": "Places the design doc at util/Seeder/CrossHarnessBridge/ (README.md, design.md, or a docs/ page inside that directory) \u2014 co-located with the concept it describes", + "passed": true, + "evidence": "'Direct answer: put the design doc at util/Seeder/CrossHarnessBridge/README.md.'" + }, + { + "text": "Cites the documentation standard's placement or decision rule as the reason, applying 'what the doc describes' as the placement input rather than 'what the code imports'", + "passed": true, + "evidence": "'Reasoning, grounded in the documentation standard's placement rule and this repo's conventions: 1. Docs live next to the code they describe. The bridge is a Seeder-owned subsystem...'" + }, + { + "text": "Does not treat cross-container imports as evidence that the doc should live at those containers' shared ancestor (e.g., repo root or a shared docs/ folder)", + "passed": true, + "evidence": "Places doc at util/Seeder/CrossHarnessBridge/README.md, explicitly rejects Seeds/docs/ and CLAUDE.md as homes, and does not propose a repo-root or shared docs/ location." + } + ], + "notes": [] + }, + { + "eval_id": 9, + "configuration": "with_skill", + "run_number": 5, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Identifies util/Seeder/CrossHarnessBridge as the artifact the design doc describes, and test/Common/Helpers/EncryptionHarness as a dependency the bridge imports rather than a subject of the doc", + "passed": true, + "evidence": "'The bridge is architecturally a Seeder concern (it lives under util/Seeder/ and wires Seeder pipeline steps into the shared harness). The EncryptionHarness under test/Common/Helpers/ is a passive helper the bridge consumes...'" + }, + { + "text": "Places the design doc at util/Seeder/CrossHarnessBridge/ (README.md, design.md, or a docs/ page inside that directory) \u2014 co-located with the concept it describes", + "passed": true, + "evidence": "'Primary: util/Seeder/CrossHarnessBridge/README.md.' Also: 'add a util/Seeder/CrossHarnessBridge/docs/ folder next to the README.'" + }, + { + "text": "Cites the documentation standard's placement or decision rule as the reason, applying 'what the doc describes' as the placement input rather than 'what the code imports'", + "passed": true, + "evidence": "'This matches the documentation standard's placement rule \u2014 docs live at their scope, as close as possible to the code they describe.'" + }, + { + "text": "Does not treat cross-container imports as evidence that the doc should live at those containers' shared ancestor (e.g., repo root or a shared docs/ folder)", + "passed": true, + "evidence": "Explicitly rejects a repo-root docs/ location: 'A repo-root docs/ directory \u2014 this repo doesn't have one; docs are colocated with code. Introducing a new top-level docs tree just for this module would fight the established layout.'" + } + ], + "notes": [] + }, + { + "eval_id": 1, + "configuration": "without_skill", + "run_number": 1, + "result": { + "pass_rate": 0.75, + "passed": 3, + "failed": 1, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Enumerates util/Seeder/Data, util/Seeder, and the repo root as documented scopes of the change, not only the nearest README", + "passed": true, + "evidence": "Surfaces enumerated section covers util/Seeder/Data/README.md, util/Seeder/README.md, util/Seeder/CLAUDE.md, and the repo-root '.claude/CLAUDE.md' \u2014 walks past the nearest README." + }, + { + "text": "Reads util/Seeder/Data/README.md before judging, and updates its generator documentation to include the new generator", + "passed": false, + "evidence": "Report explicitly says 'Data/README.md \u2014 Current Capabilities: describes user-visible cipher-content capabilities. The PassphraseGenerator produces no user-visible output today ... adding a line here would misrepresent it' and the diff has no README edit." + }, + { + "text": "For each scope with no drift, gives an explicit attestation with a reason grounded in the doc's actual content, rather than staying silent about the scope", + "passed": true, + "evidence": "Attestations section names each doc and quotes reasons from the doc content (e.g., 'Generators section: already delegates to GeneratorContext.cs remarks; no per-generator list to update')." + }, + { + "text": "Ends with a per-scope report in which every enumerated documented scope appears exactly once as either updated or verified current", + "passed": true, + "evidence": "'Docs attested for' section lists each enumerated scope exactly once marked as current." + } + ], + "notes": [] + }, + { + "eval_id": 1, + "configuration": "without_skill", + "run_number": 2, + "result": { + "pass_rate": 0.75, + "passed": 3, + "failed": 1, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Enumerates util/Seeder/Data, util/Seeder, and the repo root as documented scopes of the change, not only the nearest README", + "passed": true, + "evidence": "Documentation surfaces list includes util/Seeder/Data/README.md, util/Seeder/README.md, util/Seeder/CLAUDE.md, and the Root '.claude/CLAUDE.md' \u2014 enumerating scopes up to the repo root." + }, + { + "text": "Reads util/Seeder/Data/README.md before judging, and updates its generator documentation to include the new generator", + "passed": false, + "evidence": "Report explicitly reads Data/README.md but concludes 'Because no recipe or step invokes Passphrase ... adding a bullet here would overstate current behavior' and makes zero doc edits." + }, + { + "text": "For each scope with no drift, gives an explicit attestation with a reason grounded in the doc's actual content, rather than staying silent about the scope", + "passed": true, + "evidence": "'Files attested for' items each cite doc content (Data/README's generator overview, CLAUDE.md's 'Adding a generator' pattern and file-organization rule, etc.)." + }, + { + "text": "Ends with a per-scope report in which every enumerated documented scope appears exactly once as either updated or verified current", + "passed": true, + "evidence": "The 'Files attested for' block lists each enumerated scope exactly once, marked current." + } + ], + "notes": [] + }, + { + "eval_id": 1, + "configuration": "without_skill", + "run_number": 3, + "result": { + "pass_rate": 0.75, + "passed": 3, + "failed": 1, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Enumerates util/Seeder/Data, util/Seeder, and the repo root as documented scopes of the change, not only the nearest README", + "passed": true, + "evidence": "Scopes/surfaces enumerated section lists util/Seeder/Data/README.md, util/Seeder/README.md, util/Seeder/CLAUDE.md, plus repo-wide indexes '.claude/CLAUDE.md', '.claude/CONTRIBUTING.md' \u2014 walks up to repo root." + }, + { + "text": "Reads util/Seeder/Data/README.md before judging, and updates its generator documentation to include the new generator", + "passed": false, + "evidence": "Report reads Data/README.md fully and attests 'Current Capabilities describes emitted seed data (which is unchanged because nothing consumes _ctx.Passphrase)'; no doc edit made." + }, + { + "text": "For each scope with no drift, gives an explicit attestation with a reason grounded in the doc's actual content, rather than staying silent about the scope", + "passed": true, + "evidence": "Attestations section provides doc-grounded reasons per scope, e.g. 'the Generators section is generic, Adding a generator already points at GeneratorContext.cs remarks, and Current Capabilities describes emitted seed data'." + }, + { + "text": "Ends with a per-scope report in which every enumerated documented scope appears exactly once as either updated or verified current", + "passed": true, + "evidence": "The Attestations block enumerates each scope exactly once with a current status." + } + ], + "notes": [] + }, + { + "eval_id": 1, + "configuration": "without_skill", + "run_number": 4, + "result": { + "pass_rate": 0.75, + "passed": 3, + "failed": 1, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Enumerates util/Seeder/Data, util/Seeder, and the repo root as documented scopes of the change, not only the nearest README", + "passed": true, + "evidence": "Scopes/surfaces enumerated section lists util/Seeder/CLAUDE.md, util/Seeder/README.md, util/Seeder/Data/README.md, and 'Repo README.md and .claude-plugin/marketplace.json'." + }, + { + "text": "Reads util/Seeder/Data/README.md before judging, and updates its generator documentation to include the new generator", + "passed": false, + "evidence": "Report reads Data/README.md fully but concludes 'Individual generator classes are not enumerated ... Listing it as a capability would misrepresent state' \u2014 no doc edit made." + }, + { + "text": "For each scope with no drift, gives an explicit attestation with a reason grounded in the doc's actual content, rather than staying silent about the scope", + "passed": true, + "evidence": "Findings section explains grounded reasons per doc (delegates to GeneratorContext remarks; Current Capabilities describes emitted data; Maintenance rules do not fire without CLI/preset changes)." + }, + { + "text": "Ends with a per-scope report in which every enumerated documented scope appears exactly once as either updated or verified current", + "passed": true, + "evidence": "Docs attested for list enumerates each scope once as current." + } + ], + "notes": [] + }, + { + "eval_id": 1, + "configuration": "without_skill", + "run_number": 5, + "result": { + "pass_rate": 0.75, + "passed": 3, + "failed": 1, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Enumerates util/Seeder/Data, util/Seeder, and the repo root as documented scopes of the change, not only the nearest README", + "passed": true, + "evidence": "Doc surfaces enumerated list includes util/Seeder/Data/README.md, util/Seeder/README.md, util/Seeder/CLAUDE.md, and repo-root '.claude/CLAUDE.md'." + }, + { + "text": "Reads util/Seeder/Data/README.md before judging, and updates its generator documentation to include the new generator", + "passed": false, + "evidence": "Report reads Data/README.md thoroughly but concludes 'PassphraseGenerator is not yet consumed ... Adding a diceware passphrases bullet ... would misrepresent'; no doc edit made." + }, + { + "text": "For each scope with no drift, gives an explicit attestation with a reason grounded in the doc's actual content, rather than staying silent about the scope", + "passed": true, + "evidence": "Assessment against each surface names the specific rule/section per doc (Data/File Organization, thread-safety note line 67, Current Capabilities list, Data section deferring to Data/README)." + }, + { + "text": "Ends with a per-scope report in which every enumerated documented scope appears exactly once as either updated or verified current", + "passed": true, + "evidence": "Attestations section enumerates each scope exactly once with a current verdict." + } + ], + "notes": [] + }, + { + "eval_id": 10, + "configuration": "without_skill", + "run_number": 1, + "result": { + "pass_rate": 0.5, + "passed": 2, + "failed": 2, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Derives search terms from the renamed command verbs, changed paths, and the PR title vocabulary", + "passed": true, + "evidence": "Report enumerates search patterns derived from the diff: 'Repo-wide sweep for preset --name, --name features., --name qa., --name scale., --name individual., --name validation., and stray generate verbs'." + }, + { + "text": "Searches the contributing-docs corpus (contributing.bitwarden.com / bitwarden/contributing-docs) with those terms and reads candidate pages before judging", + "passed": false, + "evidence": "Report contains no out-of-repo discovery section; no mention of searching contributing.bitwarden.com or bitwarden/contributing-docs. All searches are repo-local." + }, + { + "text": "For any external page the rename invalidates, calls it out and states the required external-docs flow: a work item created before merge and a stale marker applied to the page", + "passed": false, + "evidence": "Report never considers external docs; the external-docs flow (work item + stale marker) is not stated anywhere." + }, + { + "text": "Also verifies the in-repo documented scopes, including cross-references from util/Seeder/CLAUDE.md to util/SeederUtility", + "passed": true, + "evidence": "Report attests 'util/Seeder/CLAUDE.md \u2014 no CLI flag references to the renamed option' and cites the CLAUDE.md maintenance rule as scoping the review." + } + ], + "notes": [] + }, + { + "eval_id": 10, + "configuration": "without_skill", + "run_number": 2, + "result": { + "pass_rate": 0.5, + "passed": 2, + "failed": 2, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Derives search terms from the renamed command verbs, changed paths, and the PR title vocabulary", + "passed": true, + "evidence": "Report shows explicit grep terms derived from the diff: 'grep -rn --include=*.md -e preset --name -e \"--name\" -e --name qa\\. -e --name scale\\. -e --name individual\\. -e --name features\\. -e --name validation\\.'" + }, + { + "text": "Searches the contributing-docs corpus (contributing.bitwarden.com / bitwarden/contributing-docs) with those terms and reads candidate pages before judging", + "passed": false, + "evidence": "Report never mentions contributing.bitwarden.com or bitwarden/contributing-docs; all searches are local to the worktree." + }, + { + "text": "For any external page the rename invalidates, calls it out and states the required external-docs flow: a work item created before merge and a stale marker applied to the page", + "passed": false, + "evidence": "Report never discusses external documentation, work items, or stale markers." + }, + { + "text": "Also verifies the in-repo documented scopes, including cross-references from util/Seeder/CLAUDE.md to util/SeederUtility", + "passed": true, + "evidence": "Report attests 'util/Seeder/CLAUDE.md \u2014 mentions preset architecture at the concept level but does not embed the --name flag; no update required' and notes CLAUDE.md's Seeds/SeederUtility scan rule guided this pass." + } + ], + "notes": [] + }, + { + "eval_id": 10, + "configuration": "without_skill", + "run_number": 3, + "result": { + "pass_rate": 0.5, + "passed": 2, + "failed": 2, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Derives search terms from the renamed command verbs, changed paths, and the PR title vocabulary", + "passed": true, + "evidence": "Report states: 'Grepped the repository for preset --name, --name in Seeder/SeederUtility markdown, and JSON-side name keys tied to the preset command.'" + }, + { + "text": "Searches the contributing-docs corpus (contributing.bitwarden.com / bitwarden/contributing-docs) with those terms and reads candidate pages before judging", + "passed": false, + "evidence": "Report never mentions contributing.bitwarden.com or bitwarden/contributing-docs; only repo-local sweeps." + }, + { + "text": "For any external page the rename invalidates, calls it out and states the required external-docs flow: a work item created before merge and a stale marker applied to the page", + "passed": false, + "evidence": "No external-docs discussion; the required flow (work item + stale marker) is not stated." + }, + { + "text": "Also verifies the in-repo documented scopes, including cross-references from util/Seeder/CLAUDE.md to util/SeederUtility", + "passed": true, + "evidence": "Report attests 'util/Seeder/CLAUDE.md \u2014 mentions the preset command generally and links to SeederUtility/README.md; contains no direct --name references, so it needs no change'." + } + ], + "notes": [] + }, + { + "eval_id": 10, + "configuration": "without_skill", + "run_number": 4, + "result": { + "pass_rate": 0.25, + "passed": 1, + "failed": 3, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Derives search terms from the renamed command verbs, changed paths, and the PR title vocabulary", + "passed": true, + "evidence": "Report notes it 'Ran a repository-wide grep for the removed flag both by itself (preset --name) and in the concrete forms that appear in examples (--name qa, --name scale, --name individual, --name features, --name validation)'." + }, + { + "text": "Searches the contributing-docs corpus (contributing.bitwarden.com / bitwarden/contributing-docs) with those terms and reads candidate pages before judging", + "passed": false, + "evidence": "Report contains no reference to contributing.bitwarden.com or bitwarden/contributing-docs; all discovery is repo-local." + }, + { + "text": "For any external page the rename invalidates, calls it out and states the required external-docs flow: a work item created before merge and a stale marker applied to the page", + "passed": false, + "evidence": "No external-docs discussion at all; the work-item + stale-marker flow is never mentioned." + }, + { + "text": "Also verifies the in-repo documented scopes, including cross-references from util/Seeder/CLAUDE.md to util/SeederUtility", + "passed": false, + "evidence": "util/Seeder/CLAUDE.md is absent from Docs read, Surfaces enumerated, and Attestations sections; only `.claude/*` skill files and hooks are referenced \u2014 the CLAUDE.md cross-reference to SeederUtility is not verified." + } + ], + "notes": [] + }, + { + "eval_id": 10, + "configuration": "without_skill", + "run_number": 5, + "result": { + "pass_rate": 0.5, + "passed": 2, + "failed": 2, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Derives search terms from the renamed command verbs, changed paths, and the PR title vocabulary", + "passed": true, + "evidence": "Report enumerates search terms and notes 'grepped *.sh, *.yml, *.yaml, *.cs, and *.json under the repo for stale usages' after listing surfaces derived from the CLI-flag rename." + }, + { + "text": "Searches the contributing-docs corpus (contributing.bitwarden.com / bitwarden/contributing-docs) with those terms and reads candidate pages before judging", + "passed": false, + "evidence": "Report never references contributing.bitwarden.com or bitwarden/contributing-docs; all discovery is confined to the local repo." + }, + { + "text": "For any external page the rename invalidates, calls it out and states the required external-docs flow: a work item created before merge and a stale marker applied to the page", + "passed": false, + "evidence": "No mention of external documentation, work items, or stale markers." + }, + { + "text": "Also verifies the in-repo documented scopes, including cross-references from util/Seeder/CLAUDE.md to util/SeederUtility", + "passed": true, + "evidence": "Report attests 'util/Seeder/CLAUDE.md \u2014 no --name references; only meta-guidance about scanning for stale flags, which is what triggered this pass' and cites the CLAUDE.md rule as the scoping rule." + } + ], + "notes": [] + }, + { + "eval_id": 2, + "configuration": "without_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and util/Seeder/CLAUDE.md before concluding", + "passed": true, + "evidence": "'Docs read' section: 'util/Seeder/CLAUDE.md - read in full. Two references... Line 48 lists IManglerService inside SeederDependencies... Line 69: ManglerService is NOT thread-safe.' And 'util/Seeder/README.md - read in full. One reference: Around line 162, under #### Services: Context-aware string mangling...'" + }, + { + "text": "Makes zero documentation edits for the behavior-preserving refactor", + "passed": true, + "evidence": "worktree.diff only touches ManglerService.cs. Report states 'Docs edited: None. The refactor is invisible at every documented layer.'" + }, + { + "text": "Ends with an explicit per-scope report where each scope is marked verified current with a one-line reason grounded in what the doc actually says", + "passed": true, + "evidence": "'Docs attested for (verified current, no edit needed)' explicitly lists util/Seeder/CLAUDE.md (line 48 IManglerService, line 69 thread-safety), util/Seeder/README.md Services paragraph, util/Seeder/Seeds/docs/presets.md, and src/Core/Auth/UserFeatures/TwoFactorAuth/readme.md, each attested current with cited doc text." + }, + { + "text": "Does not make a token or cosmetic edit to satisfy the hook", + "passed": true, + "evidence": "No doc file in the diff; report reasons 'Editing any of the surveyed docs would violate the surgical changes rule.'" + } + ], + "notes": [] + }, + { + "eval_id": 2, + "configuration": "without_skill", + "run_number": 2, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and util/Seeder/CLAUDE.md before concluding", + "passed": true, + "evidence": "'Docs I read' section: 'util/Seeder/README.md around the Services section (lines ~155-175). The paragraph on mangling describes the feature at the unique prefixes for test isolation, enabled via --mangle level.' and 'util/Seeder/CLAUDE.md (lines ~40-90). Mentions IManglerService inside the SeederDependencies bundle and the ManglerService is NOT thread-safe guidance.'" + }, + { + "text": "Makes zero documentation edits for the behavior-preserving refactor", + "passed": true, + "evidence": "Only ManglerService.cs modified in the diff. Report: 'Docs I edited: None. Every doc surface I read describes ManglerService at a level of abstraction... that the refactor does not disturb.'" + }, + { + "text": "Ends with an explicit per-scope report where each scope is marked verified current with a one-line reason grounded in what the doc actually says", + "passed": true, + "evidence": "'Attestations' block explicitly attests each surface as current with cited reasons: IManglerService.cs XML docs, ManglerService.cs class + MangleId summary, util/Seeder/README.md Services paragraph, util/Seeder/CLAUDE.md (SeederDependencies + thread-safety), and the SeederUtility/Seeds preset docs referencing --mangle." + }, + { + "text": "Does not make a token or cosmetic edit to satisfy the hook", + "passed": true, + "evidence": "No doc file in the diff; report closes with 'no docs need editing. The doc-parity hook can be cleared.'" + } + ], + "notes": [] + }, + { + "eval_id": 2, + "configuration": "without_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and util/Seeder/CLAUDE.md before concluding", + "passed": true, + "evidence": "'Docs read' section: 'util/Seeder/CLAUDE.md - Mentions IManglerService in the SeederDependencies bundle... ManglerService and SeederContext are NOT thread-safe' and 'util/Seeder/README.md - Describes the Services pattern as Context-aware string mangling for test isolation.'" + }, + { + "text": "Makes zero documentation edits for the behavior-preserving refactor", + "passed": true, + "evidence": "worktree.diff only touches ManglerService.cs. Report says 'Docs edited: None. Every doc statement about ManglerService describes public behavior that this refactor did not alter.'" + }, + { + "text": "Ends with an explicit per-scope report where each scope is marked verified current with a one-line reason grounded in what the doc actually says", + "passed": true, + "evidence": "'Docs attested for (no change needed)' lists util/Seeder/CLAUDE.md (thread-safety + SeederDependencies), util/Seeder/README.md (Services description), IManglerService.cs XML doc, SeederApi/SeederUtility/Seeds READMEs, and .claude/CLAUDE.md - each attested with a specific reason from the doc content." + }, + { + "text": "Does not make a token or cosmetic edit to satisfy the hook", + "passed": true, + "evidence": "No doc file in the diff; closing says 'No documentation is out of date as a result. The doc-parity hook can be cleared with no doc edits.'" + } + ], + "notes": [] + }, + { + "eval_id": 2, + "configuration": "without_skill", + "run_number": 4, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and util/Seeder/CLAUDE.md before concluding", + "passed": true, + "evidence": "Report cites specific line contents from util/Seeder/README.md ('line 124: Receive mangling service via DI...', 'line 162: Context-aware string mangling...') and util/Seeder/CLAUDE.md ('Line 48 lists IManglerService', 'Line 69: ManglerService... NOT thread-safe', 'Line 78 mentions --mangle')." + }, + { + "text": "Makes zero documentation edits for the behavior-preserving refactor", + "passed": true, + "evidence": "worktree.diff only shows ManglerService.cs changes. Report states 'Docs edited: None. The refactor is a pure internal reorganization inside ManglerService.Mangle.'" + }, + { + "text": "Ends with an explicit per-scope report where each scope is marked verified current with a one-line reason grounded in what the doc actually says", + "passed": true, + "evidence": "'Attestations' block goes through public API, observable behavior, _mangleMap, thread-safety, docs; earlier 'Scopes and surfaces enumerated' section attests per scope (util/Seeder/README.md, util/Seeder/CLAUDE.md, Seeds/**, XML doc surfaces) with one-line reasons grounded in the doc's actual sentences." + }, + { + "text": "Does not make a token or cosmetic edit to satisfy the hook", + "passed": true, + "evidence": "Diff contains only the .cs refactor. Report closes 'no doc file needs an edit... Doc-parity hook can be cleared.'" + } + ], + "notes": [] + }, + { + "eval_id": 2, + "configuration": "without_skill", + "run_number": 5, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and util/Seeder/CLAUDE.md before concluding", + "passed": true, + "evidence": "Report cites util/Seeder/README.md 'line 124 mentions mangling service via DI...' and 'line 162 describes... Context-aware string mangling', and util/Seeder/CLAUDE.md 'line 48 lists IManglerService... line 69 says ManglerService and SeederContext are NOT thread-safe.' Both files also appear in 'Docs Read' list." + }, + { + "text": "Makes zero documentation edits for the behavior-preserving refactor", + "passed": true, + "evidence": "worktree.diff only touches ManglerService.cs. Report explicitly says 'Docs Edited: None. The refactor is entirely below the documented surface.'" + }, + { + "text": "Ends with an explicit per-scope report where each scope is marked verified current with a one-line reason grounded in what the doc actually says", + "passed": true, + "evidence": "'Docs Attested For' block lists ManglerService.cs XML summary, IManglerService.cs XML docs, util/Seeder/README.md (lines 124, 162), util/Seeder/CLAUDE.md (lines 48, 69), SeederUtility/README.md --mangle flag, SeederApi/README.md mangleMap field, and Seeds/docs/** references, each attested current with a specific reason grounded in what the doc actually says." + }, + { + "text": "Does not make a token or cosmetic edit to satisfy the hook", + "passed": true, + "evidence": "No doc file appears in the diff; closing summary: 'no change to any name, signature, behavior, or documented characteristic... The doc-parity hook can be cleared - no doc edits needed.'" + } + ], + "notes": [] + }, + { + "eval_id": 3, + "configuration": "without_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Refuses to make the requested cosmetic or token documentation edit", + "passed": true, + "evidence": "'I did not do this. Adding a trailing newline to util/Seeder/README.md is a cosmetic change that would either be a no-op ... or leave a spurious blank line.' The diff contains no README.md edit." + }, + { + "text": "Explains why the refusal \u2014 that a real per-scope verification is the correct path through the hook", + "passed": true, + "evidence": "'Either way it papers over the hook rather than addressing what the hook is reacting to \u2014 real behavior changes in the staged diff that touch documented behavior. Per repo Rule 8 (Intellectual Honesty), I fixed the real documentation drift instead.'" + }, + { + "text": "Performs the actual verification: reads the docs at every documented ancestor scope of util/Seeder/Recipes and either updates or attests per scope", + "passed": true, + "evidence": "Enumerates util/Seeder/README.md, CLAUDE.md, verification.md, architecture.md, presets.md, fixtures.md, Seeds/README.md, Data/README.md; provides per-scope status (drifted vs no drift) and updates one XML doc." + }, + { + "text": "Ends with a per-scope report grounded in what the docs say", + "passed": true, + "evidence": "'Docs edited' and 'Not edited (attested no-drift)' sections ground each verdict in doc content, e.g., 'util/Seeder/CLAUDE.md \u2014 \"original round-robin behavior\" wording refers to pipeline Steps guarded by _density == null, not to the direct-to-DB Recipes'." + } + ], + "notes": [] + }, + { + "eval_id": 3, + "configuration": "without_skill", + "run_number": 2, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Refuses to make the requested cosmetic or token documentation edit", + "passed": true, + "evidence": "'I did not do that.' The diff contains no util/Seeder/README.md edit; only a real XML doc fix in CollectionsRecipe.cs." + }, + { + "text": "Explains why the refusal \u2014 that a real per-scope verification is the correct path through the hook", + "passed": true, + "evidence": "'The right thing to do is to check whether the docs are actually current against the real staged diff and fix any drift, not paper over the hook.'" + }, + { + "text": "Performs the actual verification: reads the docs at every documented ancestor scope of util/Seeder/Recipes and either updates or attests per scope", + "passed": true, + "evidence": "Enumerates util/Seeder/README.md, CLAUDE.md, Seeds/docs/architecture.md, presets.md, verification.md, scenarios/; grep across repo; updates the stale XML doc and attests per scope." + }, + { + "text": "Ends with a per-scope report grounded in what the docs say", + "passed": true, + "evidence": "'Surfaces attested for (no change needed, no drift found)' section grounds each verdict in doc content, e.g., 'util/Seeder/Seeds/docs/verification.md \u2014 ... describes the Pipeline Steps code path (which reads DensityProfile from preset JSON), not the CollectionsRecipe/GroupsRecipe Recipes'." + } + ], + "notes": [] + }, + { + "eval_id": 3, + "configuration": "without_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Refuses to make the requested cosmetic or token documentation edit", + "passed": true, + "evidence": "'I did NOT touch util/Seeder/README.md \u2014 no trailing newline, no other edit. Nothing in that file needed to change.' The diff shows no README.md change." + }, + { + "text": "Explains why the refusal \u2014 that a real per-scope verification is the correct path through the hook", + "passed": true, + "evidence": "'A whitespace nudge to that file is not a doc update -- it's a hook workaround, and it would leave a real staleness in the tree.' Frames refusal explicitly against silencing the hook." + }, + { + "text": "Performs the actual verification: reads the docs at every documented ancestor scope of util/Seeder/Recipes and either updates or attests per scope", + "passed": true, + "evidence": "Enumerates util/Seeder/README.md, CLAUDE.md, Seeds/docs/architecture.md, presets.md, fixtures.md, scenarios/, verification.md, Data/README.md; grep repo-wide; updates the stale XML doc and attests each scope." + }, + { + "text": "Ends with a per-scope report grounded in what the docs say", + "passed": true, + "evidence": "'Docs I attested for (read and confirmed still accurate)' lists scopes with grounded reasons, e.g., 'util/Seeder/Seeds/docs/verification.md Section 4 \"No Density (Baseline)\" -- describes the Pipeline baseline, not the recipes; unaffected by this diff'." + } + ], + "notes": [] + }, + { + "eval_id": 3, + "configuration": "without_skill", + "run_number": 4, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Refuses to make the requested cosmetic or token documentation edit", + "passed": true, + "evidence": "'I did not do that... Appending another \\n would just add a blank line \u2014 it doesn't change what the file says, and it doesn't bring any documentation into line with the code that changed. It's a hook-bypass, not a fix.' Diff contains no README.md edit." + }, + { + "text": "Explains why the refusal \u2014 that a real per-scope verification is the correct path through the hook", + "passed": true, + "evidence": "'The rest of this report is the actual verification against the staged diff, which is what the hook is asking for. If the changes are properly documented, the honest answer is to attest to that, not to poke a filler edit.'" + }, + { + "text": "Performs the actual verification: reads the docs at every documented ancestor scope of util/Seeder/Recipes and either updates or attests per scope", + "passed": true, + "evidence": "Enumerates util/Seeder/README.md, CLAUDE.md, Data/README.md, Seeds/README.md, Seeds/docs/architecture.md, verification.md, fixtures.md, presets.md; grep across repo; attests per scope." + }, + { + "text": "Ends with a per-scope report grounded in what the docs say", + "passed": true, + "evidence": "'Files I attest for' section grounds each verdict, e.g., 'util/Seeder/Seeds/docs/verification.md \u2014 accurate; the \"No Density (Baseline)\" table describes the pipeline steps, which are unchanged.'" + } + ], + "notes": [] + }, + { + "eval_id": 3, + "configuration": "without_skill", + "run_number": 5, + "result": { + "pass_rate": 0.5, + "passed": 2, + "failed": 2, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Refuses to make the requested cosmetic or token documentation edit", + "passed": false, + "evidence": "The run applied the cosmetic edit: the worktree.diff shows a '+' blank line appended at util/Seeder/README.md line 173, and the report admits 'util/Seeder/README.md \u2014 appended a single trailing blank line, exactly as the request asked'." + }, + { + "text": "Explains why the refusal \u2014 that a real per-scope verification is the correct path through the hook", + "passed": false, + "evidence": "There is no refusal to explain \u2014 the run performed the newline edit and even labels it 'a hook-satisfying whitespace edit, not documentation. I applied it as asked so the ship can go out.'" + }, + { + "text": "Performs the actual verification: reads the docs at every documented ancestor scope of util/Seeder/Recipes and either updates or attests per scope", + "passed": true, + "evidence": "Enumerates util/Seeder/README.md, CLAUDE.md, Seeds/docs/verification.md, presets.md, architecture.md, fixtures.md, scenarios/; also SeederUtility/README.md, SeederApi/README.md; provides per-scope status for each." + }, + { + "text": "Ends with a per-scope report grounded in what the docs say", + "passed": true, + "evidence": "'What I attested for' section grounds each verdict in doc content, e.g., 'util/Seeder/Seeds/docs/verification.md \u2014 every \"Collections per user\" and \"Membership shape\" row describes a density-profile preset run through the Pipeline steps; the modified recipes are not the code path those rows measure.'" + } + ], + "notes": [] + }, + { + "eval_id": 4, + "configuration": "without_skill", + "run_number": 1, + "result": { + "pass_rate": 0.0, + "passed": 0, + "failed": 4, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Names what the doc says (the deterministic claim) and what the code does (wall-clock non-determinism) as a concrete disagreement", + "passed": false, + "evidence": "No report.md exists in the outputs directory; only worktree.diff is present, and the diff contains no documentation edits or narrative naming the contradiction." + }, + { + "text": "Either edits util/Seeder/Data/README.md's determinism claim, or surfaces the contradiction with a specific recommendation \u2014 does not silently defer", + "passed": false, + "evidence": "worktree.diff contains no changes to util/Seeder/Data/README.md (or any other doc file), and there is no report to surface a recommendation \u2014 the contradiction is silently deferred." + }, + { + "text": "Still completes per-scope verification for the actual UserStatusDistributions change", + "passed": false, + "evidence": "No report.md exists; there is no per-scope narrative for the UserStatusDistributions change." + }, + { + "text": "Ends with a per-scope report", + "passed": false, + "evidence": "There is no report.md in outputs/, so no per-scope report exists." + } + ], + "notes": [] + }, + { + "eval_id": 4, + "configuration": "without_skill", + "run_number": 2, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Names what the doc says (the deterministic claim) and what the code does (wall-clock non-determinism) as a concrete disagreement", + "passed": true, + "evidence": "Report quotes the deterministic claim ('Deterministic. Seeded randomness means same org ID -> same test data -> reproducible debugging.') and states the new `Current` property that 'reads `DateTime.Now` inside its getter' contradicts it, enumerating six affected docs." + }, + { + "text": "Either edits util/Seeder/Data/README.md's determinism claim, or surfaces the contradiction with a specific recommendation \u2014 does not silently defer", + "passed": true, + "evidence": "Report surfaces the contradiction and states 'Preserve determinism (recommended, matches existing invariant). Drop the new `Current` property before merging, or make it accept an injected clock' \u2014 an explicit specific recommendation." + }, + { + "text": "Still completes per-scope verification for the actual UserStatusDistributions change", + "passed": true, + "evidence": "Report states 'util/Seeder/Data/README.md \u2014 Design Principles unchanged: the `UserStatusDistributions` tweak (the actually-described change) keeps the \"Deterministic\" claim honest for that file' and lists callers checked for stale percentage references." + }, + { + "text": "Ends with a per-scope report", + "passed": true, + "evidence": "Report has 'Docs attested (no change needed)' and 'Docs edited' sections that walk each doc scope (Data/README.md, Seeder/README.md, CLAUDE.md, presets.md, SeederUtility/README.md, bug-reproduction.md) with a verdict per scope." + } + ], + "notes": [] + }, + { + "eval_id": 4, + "configuration": "without_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Names what the doc says (the deterministic claim) and what the code does (wall-clock non-determinism) as a concrete disagreement", + "passed": true, + "evidence": "Report quotes the '**Deterministic.** Seeded randomness means same org ID \u2192 same test data \u2192 reproducible debugging' claim and states the new `Current` property 'breaks the \"same org ID \u2192 same test data\" invariant for that call site'." + }, + { + "text": "Either edits util/Seeder/Data/README.md's determinism claim, or surfaces the contradiction with a specific recommendation \u2014 does not silently defer", + "passed": true, + "evidence": "Did not edit but explicitly surfaced with a specific recommendation: 'If that addition ships in this PR, `util/Seeder/Data/README.md:11` and `util/Seeder/README.md:152` need a caveat on their unqualified \"Deterministic\" claims. If instead `Current` is meant to be reverted, split into its own PR, or reworked ... the docs stay as-is.'" + }, + { + "text": "Still completes per-scope verification for the actual UserStatusDistributions change", + "passed": true, + "evidence": "Report attests 'util/Seeder/Data/Distributions/UserStatusDistributions.cs XML doc \u2014 accurate; already updated in-diff to match the new percentages' and checks callers CreateUsersStep.cs, GeneratorContext.cs, CipherUsernameGenerator.cs for stale references." + }, + { + "text": "Ends with a per-scope report", + "passed": true, + "evidence": "Report has 'Assessment against each doc' and 'Docs attested' sections that enumerate each documented scope (Data/README.md, Seeder/README.md, CLAUDE.md, XML docs) with an explicit verdict per scope." + } + ], + "notes": [] + }, + { + "eval_id": 4, + "configuration": "without_skill", + "run_number": 4, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Names what the doc says (the deterministic claim) and what the code does (wall-clock non-determinism) as a concrete disagreement", + "passed": true, + "evidence": "Report quotes '**Deterministic.** Seeded randomness means same org ID \u2192 same test data \u2192 reproducible debugging' and states the new `Current` property that 'returns different distributions at different wall-clock times, contradicts those statements ... two seeder runs with identical inputs but different launch times can now produce different data'." + }, + { + "text": "Either edits util/Seeder/Data/README.md's determinism claim, or surfaces the contradiction with a specific recommendation \u2014 does not silently defer", + "passed": true, + "evidence": "Did not edit but explicitly surfaced with a specific recommendation: 'If the answer is \"yes, Current is intended and the principle now has an exception\", the doc changes are easy \u2014 the README bullet gets a caveat ... If the answer is \"no, this shouldn't have been added\", the docs stay as-is and the code goes.'" + }, + { + "text": "Still completes per-scope verification for the actual UserStatusDistributions change", + "passed": true, + "evidence": "The 'Change 1 \u2014 UserStatusDistributions.Realistic' section verifies 'Inline `` doc-comment on the property was updated to match ... No external doc pins a specific 85% / 5% / 5% / 5% split ... Verdict: no downstream doc drift caused by this change. Attested.'" + }, + { + "text": "Ends with a per-scope report", + "passed": true, + "evidence": "Report has 'Findings per change', 'What I edited', and 'Attested (unchanged, verified consistent)' sections walking each documented surface (README.md, CLAUDE.md, Data/README.md, XML docs, call sites) with per-scope verdicts." + } + ], + "notes": [] + }, + { + "eval_id": 4, + "configuration": "without_skill", + "run_number": 5, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Names what the doc says (the deterministic claim) and what the code does (wall-clock non-determinism) as a concrete disagreement", + "passed": true, + "evidence": "Report quotes 'Deterministic. Seeded randomness means same org ID \u2192 same test data \u2192 reproducible debugging' and states 'Adding `UsernameDistributions.Current` introduces a distribution whose output depends on wall-clock time at call site. That directly contradicts these stated invariants' \u2014 enumerating five doc surfaces." + }, + { + "text": "Either edits util/Seeder/Data/README.md's determinism claim, or surfaces the contradiction with a specific recommendation \u2014 does not silently defer", + "passed": true, + "evidence": "Did not silently defer \u2014 explicitly surfaced options: '(a) drop `Current` and keep the deterministic invariant intact, (b) keep `Current` and rewrite the Data/README, Seeder/README, Seeder/CLAUDE.md, and `Distribution.cs` class docs ... (c) reshape `Current` so it takes a clock via `GeneratorContext`'." + }, + { + "text": "Still completes per-scope verification for the actual UserStatusDistributions change", + "passed": true, + "evidence": "Report edits util/Seeder/Options/OrganizationVaultOptions.cs to match the new 80/8/7/5 split, and attests UserStatusDistributions.cs XML doc as 'Consistent with the new tuple values. No further edit needed.'" + }, + { + "text": "Ends with a per-scope report", + "passed": true, + "evidence": "'Docs attested-for' and 'Docs deliberately NOT edited' sections plus the closing summary provide a per-scope roll-up (each doc surface listed with an explicit verdict)." + } + ], + "notes": [] + }, + { + "eval_id": 5, + "configuration": "without_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Removes the policy-enterprise entry from util/Seeder/Seeds/docs/presets.md's preset catalog", + "passed": true, + "evidence": "Diff removes the policy-enterprise row and the trailing 'no ciphers' sentence from presets.md." + }, + { + "text": "Checks util/Seeder/Seeds/README.md and util/Seeder/Seeds/docs/scenarios/ for inbound references to the deleted preset and updates any that reference it", + "passed": true, + "evidence": "Report attests 'util/Seeder/Seeds/README.md ... itself only lists preset folders' and 'util/Seeder/Seeds/docs/scenarios/*.md \u2014 six scenario walkthroughs... None reference policy-enterprise or pinnacle-designs'." + }, + { + "text": "Verifies the higher scopes (util/Seeder, repo root) and reports per scope", + "passed": true, + "evidence": "Report attests 'util/Seeder/README.md \u2014 top-level Seeder overview. No mention of individual presets', 'util/Seeder/CLAUDE.md', and 'Top-level repo docs: README.md, CONTRIBUTING.md, CLAUDE.md \u2014 no references'." + }, + { + "text": "Does not leave catalog rows or links pointing at the removed preset", + "passed": true, + "evidence": "Report notes 'full-worktree grep for policy-enterprise, pinnacle, and Pinnacle returns no matches' after edits; diff shows both catalog row and follow-up sentence removed." + } + ], + "notes": [] + }, + { + "eval_id": 5, + "configuration": "without_skill", + "run_number": 2, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Removes the policy-enterprise entry from util/Seeder/Seeds/docs/presets.md's preset catalog", + "passed": true, + "evidence": "Diff removes both the `policy-enterprise` row and the follow-up 'no ciphers' sentence; report explicitly says 'Removed the policy-enterprise ... row' and 'Removed the follow-up sentence'." + }, + { + "text": "Checks util/Seeder/Seeds/README.md and util/Seeder/Seeds/docs/scenarios/ for inbound references to the deleted preset and updates any that reference it", + "passed": true, + "evidence": "Report attests 'util/Seeder/Seeds/README.md line 21 \u2014 the features/ folder row says the folder is for SSO, TDE, policies... remains truthful. No edit.' and 'scenarios/ set \u2014 spot-checked via grep; no matches'." + }, + { + "text": "Verifies the higher scopes (util/Seeder, repo root) and reports per scope", + "passed": true, + "evidence": "Report attests 'util/Seeder/CLAUDE.md line 99 \u2014 lists preset subfolders... still accurate. No edit' and 'Repo top-level (README.md, CONTRIBUTING.md, SECURITY.md, etc.) \u2014 no references'." + }, + { + "text": "Does not leave catalog rows or links pointing at the removed preset", + "passed": true, + "evidence": "Report says 'Post-edit re-run of the first sweep returned zero hits' and diff removes both catalog row and follow-up sentence." + } + ], + "notes": [] + }, + { + "eval_id": 5, + "configuration": "without_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Removes the policy-enterprise entry from util/Seeder/Seeds/docs/presets.md's preset catalog", + "passed": true, + "evidence": "Diff removes the `policy-enterprise` row from the Features preset table and the follow-up 'no ciphers' sentence." + }, + { + "text": "Checks util/Seeder/Seeds/README.md and util/Seeder/Seeds/docs/scenarios/ for inbound references to the deleted preset and updates any that reference it", + "passed": true, + "evidence": "Report attests 'util/Seeder/Seeds/README.md \u2014 top-level Seeds index; uses features.sso-enterprise as example... No update' and 'util/Seeder/Seeds/docs/scenarios/*.md \u2014 scanned all seven scenarios; none reference the deleted preset'." + }, + { + "text": "Verifies the higher scopes (util/Seeder, repo root) and reports per scope", + "passed": true, + "evidence": "Report attests 'util/Seeder/README.md ... no mention of the deleted preset. No update', 'util/Seeder/CLAUDE.md ... No update', and 'Root README.md, CONTRIBUTING.md, and everything under dev/ \u2014 no references found'." + }, + { + "text": "Does not leave catalog rows or links pointing at the removed preset", + "passed": true, + "evidence": "Report states 'both mentions are now removed' and diff shows all references removed; report also confirms 'no other documentation surface, scenario, README, CLAUDE.md, or test hard-codes the deleted names'." + } + ], + "notes": [] + }, + { + "eval_id": 5, + "configuration": "without_skill", + "run_number": 4, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Removes the policy-enterprise entry from util/Seeder/Seeds/docs/presets.md's preset catalog", + "passed": true, + "evidence": "Diff removes the `policy-enterprise` row and the trailing 'no ciphers' sentence from presets.md; report says 'the Features table row for policy-enterprise ... was removed'." + }, + { + "text": "Checks util/Seeder/Seeds/README.md and util/Seeder/Seeds/docs/scenarios/ for inbound references to the deleted preset and updates any that reference it", + "passed": true, + "evidence": "Report attests 'util/Seeder/Seeds/README.md \u2014 the preset-folder table ... Still accurate. No update' and lists 'Seeds/docs/scenarios/*.md' among attested docs with no matches." + }, + { + "text": "Verifies the higher scopes (util/Seeder, repo root) and reports per scope", + "passed": true, + "evidence": "Report attests 'util/Seeder/README.md \u2014 read in full ... no per-preset content', 'util/Seeder/CLAUDE.md \u2014 spot-checked ... only unrelated matches', and repo-root docs verified via grep." + }, + { + "text": "Does not leave catalog rows or links pointing at the removed preset", + "passed": true, + "evidence": "Report states 'grep -rn \"policy-enterprise\\|pinnacle\" across the worktree ... returns no matches, confirming no dangling references remain'." + } + ], + "notes": [] + }, + { + "eval_id": 5, + "configuration": "without_skill", + "run_number": 5, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Removes the policy-enterprise entry from util/Seeder/Seeds/docs/presets.md's preset catalog", + "passed": true, + "evidence": "Diff removes the `policy-enterprise` row from the Features preset table and the follow-up 'no ciphers' sentence." + }, + { + "text": "Checks util/Seeder/Seeds/README.md and util/Seeder/Seeds/docs/scenarios/ for inbound references to the deleted preset and updates any that reference it", + "passed": true, + "evidence": "Report checked Seeds/README.md and updated the features/ folder description (removing 'policies' from 'SSO, TDE, policies'); scenarios attested: 'scenarios don't map to features.policy-enterprise. No update.'" + }, + { + "text": "Verifies the higher scopes (util/Seeder, repo root) and reports per scope", + "passed": true, + "evidence": "Report attests 'util/Seeder/CLAUDE.md \u2014 the two policy/policies matches are about collection-management settings, not the deleted preset. No update' and 'util/Seeder/README.md ... no references. No update', plus 'Repo root (README.md, CONTRIBUTING.md) \u2014 no seeder/preset content. No update'." + }, + { + "text": "Does not leave catalog rows or links pointing at the removed preset", + "passed": true, + "evidence": "Report concludes 'No other doc surface in the Seeder tree, or elsewhere in the repo, mentions policy-enterprise, pinnacle-designs, or the removed roster/policy combination'; diff removes catalog row, follow-up sentence, and the corresponding README description." + } + ], + "notes": [] + }, + { + "eval_id": 6, + "configuration": "without_skill", + "run_number": 1, + "result": { + "pass_rate": 0.6, + "passed": 3, + "failed": 2, + "total": 5, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and identifies that its `attachmentVersion` axis table enumerates `v0 | v1 | v2` and the cipher/attachment invariant needs revisiting for `v3`", + "passed": false, + "evidence": "Report reads README.md and notes it 'lists `v0 \u00b7 v1 \u00b7 v2` as `attachmentVersion` values', but attests it current on the basis that fixture surface is unchanged: 'every claim in these two documents remains literally true for the values it enumerates'." + }, + { + "text": "Updates util/Seeder/README.md's attachmentVersion enumeration and invariant rather than only attesting", + "passed": false, + "evidence": "Report explicitly says 'Docs I edited: None.' and worktree.diff shows no changes to README.md." + }, + { + "text": "Enumerates the changed source files (Pipeline/, Factories/) as below-component source-embedded surfaces in Step 2, distinct from the tree-level README scopes", + "passed": true, + "evidence": "Section 2 lists 'XML doc comments on the touched classes' as the first bullet, distinct from later 'Repo-facing docs under util/Seeder/' bullets that enumerate README.md, CLAUDE.md, and the Seeds docs." + }, + { + "text": "Also checks util/Seeder/Seeds/ documentation for references to the attachmentVersion key", + "passed": true, + "evidence": "Report lists and reads util/Seeder/Seeds/docs/presets.md, Seeds/docs/scenarios/encryption-modes.md, encryption-modes-org.md, blob-migration.md, and Seeds/schemas/cipher.schema.json." + }, + { + "text": "Does not treat the already-updated XML doc comments as sufficient for the higher-scope READMEs", + "passed": true, + "evidence": "Report explicitly reads util/Seeder/README.md and CLAUDE.md, providing grounded reasoning on each, rather than stopping at XML doc surface." + } + ], + "notes": [] + }, + { + "eval_id": 6, + "configuration": "without_skill", + "run_number": 2, + "result": { + "pass_rate": 0.6, + "passed": 3, + "failed": 2, + "total": 5, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and identifies that its `attachmentVersion` axis table enumerates `v0 | v1 | v2` and the cipher/attachment invariant needs revisiting for `v3`", + "passed": false, + "evidence": "Report reads README.md but concludes 'the fixture-level docs and Rust FFI docs are still accurate for what the seeder pipeline actually accepts end-to-end today and shouldn't be updated until v3 is wired through'. Does not identify the enumeration/invariant needs revisiting." + }, + { + "text": "Updates util/Seeder/README.md's attachmentVersion enumeration and invariant rather than only attesting", + "passed": false, + "evidence": "Report explicitly states 'Edits made: None' and defers README.md updates. worktree.diff shows no README.md changes." + }, + { + "text": "Enumerates the changed source files (Pipeline/, Factories/) as below-component source-embedded surfaces in Step 2, distinct from the tree-level README scopes", + "passed": true, + "evidence": "Report separates 'On the touched classes (author's XML/`` updates)' section from 'Adjacent doc surfaces walked for drift' section, listing all three touched files with their XML doc surfaces distinctly." + }, + { + "text": "Also checks util/Seeder/Seeds/ documentation for references to the attachmentVersion key", + "passed": true, + "evidence": "Report reads util/Seeder/Seeds/docs/scenarios/encryption-modes.md, encryption-modes-org.md, Seeds/docs/presets.md, Seeds/schemas/cipher.schema.json." + }, + { + "text": "Does not treat the already-updated XML doc comments as sufficient for the higher-scope READMEs", + "passed": true, + "evidence": "Report examines util/Seeder/CLAUDE.md, README.md, and the wider Seeder tree; even notes stale enum-membership sentence in CLAUDE.md rather than stopping at XML docs." + } + ], + "notes": [] + }, + { + "eval_id": 6, + "configuration": "without_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 5, + "failed": 0, + "total": 5, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and identifies that its `attachmentVersion` axis table enumerates `v0 | v1 | v2` and the cipher/attachment invariant needs revisiting for `v3`", + "passed": true, + "evidence": "Report reads README.md and identifies the 'Encryption Schemes' table + invariant + not-account-Encryption-V1/V2 aside, then updates them: 'extended the attachment-scheme row of the Encryption Schemes table to include v3... extended the invariant sentence to add the v3 rule'." + }, + { + "text": "Updates util/Seeder/README.md's attachmentVersion enumeration and invariant rather than only attesting", + "passed": true, + "evidence": "worktree.diff shows README.md updated: table now includes 'v0 \u00b7 v1 \u00b7 v2 \u00b7 v3' and invariant now reads 'v3 requires an org-owned cipher (personal-vault ciphers cannot host v3)'." + }, + { + "text": "Enumerates the changed source files (Pipeline/, Factories/) as below-component source-embedded surfaces in Step 2, distinct from the tree-level README scopes", + "passed": true, + "evidence": "Report has explicit 'XML doc comments on touched classes/members' bullet listing all three touched files, distinct from 'Repo-level prose docs' bullet listing README.md, CLAUDE.md, etc." + }, + { + "text": "Also checks util/Seeder/Seeds/ documentation for references to the attachmentVersion key", + "passed": true, + "evidence": "Report lists and updates Seeds documentation: 'Seeds/docs/scenarios/encryption-modes-org.md', 'Seeds/docs/presets.md', and reads 'Seeds/docs/scenarios/encryption-modes.md', 'Seeds/docs/architecture.md', 'Seeds/docs/fixtures.md', 'Seeds/docs/verification.md', 'Seeds/README.md'." + }, + { + "text": "Does not treat the already-updated XML doc comments as sufficient for the higher-scope READMEs", + "passed": true, + "evidence": "Report explicitly performs additional edits on README.md, CLAUDE.md, Seeds/docs/presets.md, and Seeds/docs/scenarios/encryption-modes-org.md, going beyond the pre-updated XML docs." + } + ], + "notes": [] + }, + { + "eval_id": 6, + "configuration": "without_skill", + "run_number": 4, + "result": { + "pass_rate": 0.6, + "passed": 3, + "failed": 2, + "total": 5, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and identifies that its `attachmentVersion` axis table enumerates `v0 | v1 | v2` and the cipher/attachment invariant needs revisiting for `v3`", + "passed": false, + "evidence": "Report reads README.md but attests it current: 'describes the fixture-facing `attachmentVersion` surface, which still only exposes v0/v1/v2 (see next section). Accurate at that layer.' Does not identify enumeration/invariant needs revisiting." + }, + { + "text": "Updates util/Seeder/README.md's attachmentVersion enumeration and invariant rather than only attesting", + "passed": false, + "evidence": "Report says 'Docs I edited: None' and worktree.diff shows no README.md changes." + }, + { + "text": "Enumerates the changed source files (Pipeline/, Factories/) as below-component source-embedded surfaces in Step 2, distinct from the tree-level README scopes", + "passed": true, + "evidence": "Report's 'Surfaces found' list places the three touched files first with their XML doc distinctly, before enumerating the higher-scope README.md and CLAUDE.md." + }, + { + "text": "Also checks util/Seeder/Seeds/ documentation for references to the attachmentVersion key", + "passed": true, + "evidence": "Report lists and reads Seeds/schemas/cipher.schema.json, Seeds/docs/presets.md, Seeds/docs/scenarios/encryption-modes.md and encryption-modes-org.md." + }, + { + "text": "Does not treat the already-updated XML doc comments as sufficient for the higher-scope READMEs", + "passed": true, + "evidence": "Report reads and attests higher-scope README.md and CLAUDE.md separately from the XML docs; also flags CLAUDE.md line 163 wire-mapping as considered-for-edit." + } + ], + "notes": [] + }, + { + "eval_id": 6, + "configuration": "without_skill", + "run_number": 5, + "result": { + "pass_rate": 0.6, + "passed": 3, + "failed": 2, + "total": 5, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Reads util/Seeder/README.md and identifies that its `attachmentVersion` axis table enumerates `v0 | v1 | v2` and the cipher/attachment invariant needs revisiting for `v3`", + "passed": false, + "evidence": "Report reads README.md and only updates the axis-label callout for v3 but attests the main axis table and invariant as current: identifies fixture-scenario docs 'describe *what the fixtures actually contain*, and no fixture currently includes v3' \u2014 treats the main enumeration and invariant as not needing revisiting." + }, + { + "text": "Updates util/Seeder/README.md's attachmentVersion enumeration and invariant rather than only attesting", + "passed": false, + "evidence": "worktree.diff shows README.md updated only in the 'Not the same as account Encryption V1/V2' callout (v0/v1/v2 \u2192 v0/v1/v2/v3). The main axis table and invariant sentences are NOT updated to include v3." + }, + { + "text": "Enumerates the changed source files (Pipeline/, Factories/) as below-component source-embedded surfaces in Step 2, distinct from the tree-level README scopes", + "passed": true, + "evidence": "Report's doc-surfaces table lists 'Touched-class XML docs (attested)' as the first row with the three touched files, distinct from later 'Repo-level Seeder README' and 'Seeder CLAUDE.md' rows." + }, + { + "text": "Also checks util/Seeder/Seeds/ documentation for references to the attachmentVersion key", + "passed": true, + "evidence": "Report lists Seeds/docs/scenarios/encryption-modes.md, encryption-modes-org.md, Seeds/docs/presets.md, and Seeds/schemas/cipher.schema.json as read." + }, + { + "text": "Does not treat the already-updated XML doc comments as sufficient for the higher-scope READMEs", + "passed": true, + "evidence": "Report updates higher-scope CLAUDE.md (four touch points) and part of README.md, going beyond the pre-updated XML docs." + } + ], + "notes": [] + }, + { + "eval_id": 7, + "configuration": "without_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 3, + "failed": 0, + "total": 3, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Treats util/Seeder/CLAUDE.md as documentation in scope and updates its `dotnet test` command(s) to the new path", + "passed": true, + "evidence": "Report lists 'util/Seeder/CLAUDE.md' as edited with 'Line 16: dotnet test test/SeederApi.IntegrationTest/ -> dotnet test test/Seeder.IntegrationTest/' etc. Diff confirms the CLAUDE.md updates." + }, + { + "text": "Checks the other Seeder READMEs for references to the old test project path and updates any found", + "passed": true, + "evidence": "Report says 'util/Seeder/README.md, util/SeederApi/README.md, util/SeederUtility/README.md - checked for stale references; all their SeederApi mentions refer to the utility project... not the renamed test project. No edits needed.'" + }, + { + "text": "Reports per scope, marking each scope as either updated or verified current", + "passed": true, + "evidence": "Report has 'Docs edited' and 'Docs attested (read, no edit needed)' sections listing each scope with its status." + } + ], + "notes": [] + }, + { + "eval_id": 7, + "configuration": "without_skill", + "run_number": 2, + "result": { + "pass_rate": 1.0, + "passed": 3, + "failed": 0, + "total": 3, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Treats util/Seeder/CLAUDE.md as documentation in scope and updates its `dotnet test` command(s) to the new path", + "passed": true, + "evidence": "Report says 'util/Seeder/CLAUDE.md - Quick Reference ## Commands block ran tests via dotnet test test/SeederApi.IntegrationTest/... Line 16/19 updated to dotnet test test/Seeder.IntegrationTest/'. Diff confirms the edits." + }, + { + "text": "Checks the other Seeder READMEs for references to the old test project path and updates any found", + "passed": true, + "evidence": "Report says 'util/Seeder/README.md, util/SeederApi/README.md, util/SeederUtility/README.md -- verified they do not name the integration-test project. No change needed.' Also grepped util/Seeder/Seeds/docs/**." + }, + { + "text": "Reports per scope, marking each scope as either updated or verified current", + "passed": true, + "evidence": "Report has 'Docs edited' and 'Docs attested (no edit needed)' sections listing each scope with its status. The closing summary also enumerates per-scope status." + } + ], + "notes": [] + }, + { + "eval_id": 7, + "configuration": "without_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 3, + "failed": 0, + "total": 3, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Treats util/Seeder/CLAUDE.md as documentation in scope and updates its `dotnet test` command(s) to the new path", + "passed": true, + "evidence": "Report lists 'util/Seeder/CLAUDE.md line 16: dotnet test test/SeederApi.IntegrationTest/ -> dotnet test test/Seeder.IntegrationTest/'. Diff confirms both dotnet test commands and the Steps/ prose reference updated." + }, + { + "text": "Checks the other Seeder READMEs for references to the old test project path and updates any found", + "passed": true, + "evidence": "Report explicitly lists 'util/Seeder/README.md, util/Seeder/Data/README.md, util/Seeder/Seeds/README.md, util/Seeder/Seeds/docs/*.md... -- no references to the integration-test project' and 'util/SeederApi/README.md, util/SeederUtility/README.md -- ... Still accurate.'" + }, + { + "text": "Reports per scope, marking each scope as either updated or verified current", + "passed": true, + "evidence": "Report has 'Live documentation references found (needed editing)', 'Historical / attested references', and 'Docs read and verified current' sections listing each scope with its status." + } + ], + "notes": [] + }, + { + "eval_id": 7, + "configuration": "without_skill", + "run_number": 4, + "result": { + "pass_rate": 1.0, + "passed": 3, + "failed": 0, + "total": 3, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Treats util/Seeder/CLAUDE.md as documentation in scope and updates its `dotnet test` command(s) to the new path", + "passed": true, + "evidence": "Report lists 'util/Seeder/CLAUDE.md - Line 16 & 19: dotnet test test/SeederApi.IntegrationTest/ -> .../Seeder.IntegrationTest/'. Diff confirms updates to both commands." + }, + { + "text": "Checks the other Seeder READMEs for references to the old test project path and updates any found", + "passed": true, + "evidence": "Report says 'util/Seeder/README.md - Seeder library overview (mentions only SeederApi as an app name, not the test project - no stale references)' and 'util/SeederApi/README.md... - no stale references'." + }, + { + "text": "Reports per scope, marking each scope as either updated or verified current", + "passed": true, + "evidence": "Report has 'Docs edited' and 'Attested (no change needed)' sections listing every doc scope with its status." + } + ], + "notes": [] + }, + { + "eval_id": 7, + "configuration": "without_skill", + "run_number": 5, + "result": { + "pass_rate": 1.0, + "passed": 3, + "failed": 0, + "total": 3, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Treats util/Seeder/CLAUDE.md as documentation in scope and updates its `dotnet test` command(s) to the new path", + "passed": true, + "evidence": "Report lists 'util/Seeder/CLAUDE.md: dotnet test test/SeederApi.IntegrationTest/ -> dotnet test test/Seeder.IntegrationTest/' plus the --filter variant. Diff confirms both commands updated." + }, + { + "text": "Checks the other Seeder READMEs for references to the old test project path and updates any found", + "passed": true, + "evidence": "Report says 'util/Seeder/README.md, util/SeederApi/README.md, util/SeederUtility/README.md, and every doc under util/Seeder/Seeds/**: no mentions of the moved project. No changes needed.'" + }, + { + "text": "Reports per scope, marking each scope as either updated or verified current", + "passed": true, + "evidence": "Report has 'Edits Made' and 'Attested (Left As-Is)' sections that enumerate every scope and mark it either 'updated' or 'no changes needed'." + } + ], + "notes": [] + }, + { + "eval_id": 8, + "configuration": "without_skill", + "run_number": 1, + "result": { + "pass_rate": 0.75, + "passed": 3, + "failed": 1, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Recognizes the diff as doc-comment-only inside a source file misclassified as code by the Stop hook", + "passed": true, + "evidence": "Report: 'The doc-parity Stop hook watches `.cs` files as code, but the actual diff is a **doc-comment-only edit**.'" + }, + { + "text": "Dismisses the check with a one-line reason naming the false-positive class", + "passed": true, + "evidence": "Closing summary: 'Documentation parity: **dismissed** \u2014 doc-comment-only edit inside `///` blocks, no documentation obligation and no ancestor scope drifted.'" + }, + { + "text": "Does not walk the per-scope verification tree after dismissing", + "passed": false, + "evidence": "Explicitly walked the tree: 'I still did a light sanity walk over the ancestor scopes' \u2014 reviewed util/Seeder/Data/README.md, util/Seeder/README.md, util/Seeder/CLAUDE.md, and repo root README.md, and issued attestations for each." + }, + { + "text": "Does not make any additional documentation edits", + "passed": true, + "evidence": "worktree.diff shows only the original two doc-comment hunks in GeneratorContext.cs; report says 'Docs edited: None.'" + } + ], + "notes": [] + }, + { + "eval_id": 8, + "configuration": "without_skill", + "run_number": 2, + "result": { + "pass_rate": 0.25, + "passed": 1, + "failed": 3, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Recognizes the diff as doc-comment-only inside a source file misclassified as code by the Stop hook", + "passed": true, + "evidence": "Report: 'No executable code changed... The staged diff touches only XML doc comments in util/Seeder/Data/GeneratorContext.cs.'" + }, + { + "text": "Dismisses the check with a one-line reason naming the false-positive class", + "passed": false, + "evidence": "Report never dismisses; instead treats it as a drift risk and proceeds with full verification and an edit. Closing summary describes making an edit to bring the README into line, not dismissing." + }, + { + "text": "Does not walk the per-scope verification tree after dismissing", + "passed": false, + "evidence": "Report enumerates all files that name GeneratorContext (8 files) and greps for multiple phrases, then reads util/Seeder/Data/README.md, util/Seeder/CLAUDE.md, and other surfaces in full." + }, + { + "text": "Does not make any additional documentation edits", + "passed": false, + "evidence": "worktree.diff shows a second hunk editing util/Seeder/Data/README.md line 16 changing 'lazy-initializes' to 'lazily initializes'; report also explicitly says 'Docs edited: util/Seeder/Data/README.md, line 16.'" + } + ], + "notes": [] + }, + { + "eval_id": 8, + "configuration": "without_skill", + "run_number": 3, + "result": { + "pass_rate": 0.5, + "passed": 2, + "failed": 2, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Recognizes the diff as doc-comment-only inside a source file misclassified as code by the Stop hook", + "passed": true, + "evidence": "Report: 'Both hunks live entirely inside `///` comment lines... this is a documentation-only change with no behavioral or API surface change.' Closing: 'The hook fire is a false positive.'" + }, + { + "text": "Dismisses the check with a one-line reason naming the false-positive class", + "passed": false, + "evidence": "No one-line dismissal upfront. The report performs full multi-step verification with scope enumeration, doc reads, and attestations before concluding 'The hook fire is a false positive' at the end." + }, + { + "text": "Does not walk the per-scope verification tree after dismissing", + "passed": false, + "evidence": "Report explicitly enumerates 'Scopes / surfaces enumerated' with direct call sites, project-level guidance files, and repo-wide markdown; runs multiple grep commands; reads util/Seeder/Data/README.md and util/Seeder/CLAUDE.md; issues attestations for each surface." + }, + { + "text": "Does not make any additional documentation edits", + "passed": true, + "evidence": "Report: 'Files edited: None.' and worktree.diff shows only the original two hunks in GeneratorContext.cs." + } + ], + "notes": [] + }, + { + "eval_id": 8, + "configuration": "without_skill", + "run_number": 4, + "result": { + "pass_rate": 0.5, + "passed": 2, + "failed": 2, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Recognizes the diff as doc-comment-only inside a source file misclassified as code by the Stop hook", + "passed": true, + "evidence": "Report: 'Both hunks are inside `///` XML-doc blocks. No signature, method body, type layout, attribute, or namespace changed.' Verdict: 'The hook alert is a false positive: this is a docs-only, semantically-neutral wording refinement.'" + }, + { + "text": "Dismisses the check with a one-line reason naming the false-positive class", + "passed": false, + "evidence": "Report does not dismiss early with a one-line reason. It performs full verification, then declares false positive at the end after a lengthy verdict section." + }, + { + "text": "Does not walk the per-scope verification tree after dismissing", + "passed": false, + "evidence": "Report walks the tree extensively: 'Scopes / surfaces enumerated' section lists nearest folder README, project-level guidance, repo-wide markdown; runs grep -rn commands; reads util/Seeder/Data/README.md and util/Seeder/CLAUDE.md in full; issues attestations." + }, + { + "text": "Does not make any additional documentation edits", + "passed": true, + "evidence": "Report: 'Docs edited: None' and 'Diff at end of run is identical to the diff at start of run'. worktree.diff shows only original two hunks." + } + ], + "notes": [] + }, + { + "eval_id": 8, + "configuration": "without_skill", + "run_number": 5, + "result": { + "pass_rate": 0.5, + "passed": 2, + "failed": 2, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Recognizes the diff as doc-comment-only inside a source file misclassified as code by the Stop hook", + "passed": true, + "evidence": "Report: 'The full diff is exactly two edits, both inside `///` XML-doc blocks... No executable code lines change.' Closing: 'This is a false-positive fire.'" + }, + { + "text": "Dismisses the check with a one-line reason naming the false-positive class", + "passed": false, + "evidence": "Report does not dismiss early. It performs full verification with 8 doc surfaces inspected, grep-based drift search, and per-surface attestations before concluding false positive." + }, + { + "text": "Does not walk the per-scope verification tree after dismissing", + "passed": false, + "evidence": "Report walks tree extensively: enumerates 8 doc files (util/Seeder/README.md, CLAUDE.md, Data/README.md, Seeds/README.md, four Seeds/docs/*.md files); runs multiple greps across the worktree; issues attestations 'Docs Attested For (No Change Needed)'." + }, + { + "text": "Does not make any additional documentation edits", + "passed": true, + "evidence": "Report: 'Docs Edited: None.' and 'I deliberately did **not** rewrite util/Seeder/Data/README.md:16'. worktree.diff shows only the original two hunks." + } + ], + "notes": [] + }, + { + "eval_id": 9, + "configuration": "without_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Identifies util/Seeder/CrossHarnessBridge as the artifact the design doc describes, and test/Common/Helpers/EncryptionHarness as a dependency the bridge imports rather than a subject of the doc", + "passed": true, + "evidence": "'Ownership. The bridge is a Seeder concern (namespace Bit.Seeder.CrossHarnessBridge; XML doc says \"Bridges Seeder pipeline steps to the shared encryption harness\"). The harness is its dependency. Design docs belong with the owning module.'" + }, + { + "text": "Places the design doc at util/Seeder/CrossHarnessBridge/ (README.md, design.md, or a docs/ page inside that directory) \u2014 co-located with the concept it describes", + "passed": true, + "evidence": "'Primary home: util/Seeder/CrossHarnessBridge/README.md - colocated with the module.' Also: 'split it into util/Seeder/CrossHarnessBridge/docs/design.md and let the README link to it.'" + }, + { + "text": "Cites the documentation standard's placement or decision rule as the reason, applying 'what the doc describes' as the placement input rather than 'what the code imports'", + "passed": true, + "evidence": "'Design docs belong with the owning module... Every self-contained Seeder subdomain has its own README next to its code... CrossHarnessBridge/ is the same shape and should follow the same rule.' Applies co-location rule based on ownership/what the doc describes." + }, + { + "text": "Does not treat cross-container imports as evidence that the doc should live at those containers' shared ancestor (e.g., repo root or a shared docs/ folder)", + "passed": true, + "evidence": "'Don't create a repo-level docs/ or architecture/ folder. The repo doesn't use that pattern.'" + } + ], + "notes": [] + }, + { + "eval_id": 9, + "configuration": "without_skill", + "run_number": 2, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Identifies util/Seeder/CrossHarnessBridge as the artifact the design doc describes, and test/Common/Helpers/EncryptionHarness as a dependency the bridge imports rather than a subject of the doc", + "passed": true, + "evidence": "'The CrossHarnessBridge is a Seeder-side component; its dependency on test/Common/Helpers/EncryptionHarness/ is inward. The EncryptionHarness itself is a thin data holder - the design is in the bridge that consumes it.'" + }, + { + "text": "Places the design doc at util/Seeder/CrossHarnessBridge/ (README.md, design.md, or a docs/ page inside that directory) \u2014 co-located with the concept it describes", + "passed": true, + "evidence": "'Primary home: util/Seeder/CrossHarnessBridge/README.md (a new file, colocated with the code).'" + }, + { + "text": "Cites the documentation standard's placement or decision rule as the reason, applying 'what the doc describes' as the placement input rather than 'what the code imports'", + "passed": true, + "evidence": "'Colocating the doc with the consuming code matches the Seeder's existing pattern (util/Seeder/Data/README.md, util/Seeder/Seeds/README.md).' Rule applied: docs live with the module the doc describes." + }, + { + "text": "Does not treat cross-container imports as evidence that the doc should live at those containers' shared ancestor (e.g., repo root or a shared docs/ folder)", + "passed": true, + "evidence": "Places doc inside util/Seeder/CrossHarnessBridge/ (not at any shared ancestor of util/ and test/). Also does not suggest a repo-root docs location." + } + ], + "notes": [] + }, + { + "eval_id": 9, + "configuration": "without_skill", + "run_number": 3, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Identifies util/Seeder/CrossHarnessBridge as the artifact the design doc describes, and test/Common/Helpers/EncryptionHarness as a dependency the bridge imports rather than a subject of the doc", + "passed": true, + "evidence": "'Ownership. The bridge is a Seeder subsystem (namespace Bit.Seeder.CrossHarnessBridge). The EncryptionHarness is a dependency it consumes, not the other component being designed. Docs belong with the owning module.'" + }, + { + "text": "Places the design doc at util/Seeder/CrossHarnessBridge/ (README.md, design.md, or a docs/ page inside that directory) \u2014 co-located with the concept it describes", + "passed": true, + "evidence": "'Put the design doc at: util/Seeder/CrossHarnessBridge/README.md'" + }, + { + "text": "Cites the documentation standard's placement or decision rule as the reason, applying 'what the doc describes' as the placement input rather than 'what the code imports'", + "passed": true, + "evidence": "'Docs belong with the owning module... Every other first-class Seeder-family subsystem documents itself with a colocated README.md.' Applies the co-location placement rule based on what the doc describes." + }, + { + "text": "Does not treat cross-container imports as evidence that the doc should live at those containers' shared ancestor (e.g., repo root or a shared docs/ folder)", + "passed": true, + "evidence": "Enumerates 'A repo-top-level docs area / ADR \u2014 no matching convention exists in this repo' and does not choose it; places the doc inside util/Seeder/CrossHarnessBridge/." + } + ], + "notes": [] + }, + { + "eval_id": 9, + "configuration": "without_skill", + "run_number": 4, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Identifies util/Seeder/CrossHarnessBridge as the artifact the design doc describes, and test/Common/Helpers/EncryptionHarness as a dependency the bridge imports rather than a subject of the doc", + "passed": true, + "evidence": "'The test/Common/Helpers/EncryptionHarness/ files are a dependency of the bridge, not its topic... the bridge's doc belongs with the bridge.' Also: 'the bridge lives in Seeder and the design is a Seeder concern; the harness is the dependency, not the topic.'" + }, + { + "text": "Places the design doc at util/Seeder/CrossHarnessBridge/ (README.md, design.md, or a docs/ page inside that directory) \u2014 co-located with the concept it describes", + "passed": true, + "evidence": "'Primary: util/Seeder/CrossHarnessBridge/README.md \u2014 colocated with the code, matching the folder-README precedent...'" + }, + { + "text": "Cites the documentation standard's placement or decision rule as the reason, applying 'what the doc describes' as the placement input rather than 'what the code imports'", + "passed": true, + "evidence": "'colocated with the code, matching the folder-README precedent set by util/Seeder/Data/README.md and util/Seeder/Seeds/README.md. This is the natural \"how it works\" home.' Applies co-location rule based on what the doc describes." + }, + { + "text": "Does not treat cross-container imports as evidence that the doc should live at those containers' shared ancestor (e.g., repo root or a shared docs/ folder)", + "passed": true, + "evidence": "'create a new top-level docs/ folder \u2014 no such convention exists in this repo.' Also: 'There is no repo-wide docs/design/, docs/architecture/, or adr/ folder in bitwarden/server.'" + } + ], + "notes": [] + }, + { + "eval_id": 9, + "configuration": "without_skill", + "run_number": 5, + "result": { + "pass_rate": 1.0, + "passed": 4, + "failed": 0, + "total": 4, + "time_seconds": 0.0, + "tokens": 0, + "tool_calls": 0, + "errors": 0 + }, + "expectations": [ + { + "text": "Identifies util/Seeder/CrossHarnessBridge as the artifact the design doc describes, and test/Common/Helpers/EncryptionHarness as a dependency the bridge imports rather than a subject of the doc", + "passed": true, + "evidence": "'Ownership sits on the Seeder side. The bridge class lives in Bit.Seeder.CrossHarnessBridge; the test helper it depends on is a passive collaborator. Design docs belong with the owning module.'" + }, + { + "text": "Places the design doc at util/Seeder/CrossHarnessBridge/ (README.md, design.md, or a docs/ page inside that directory) \u2014 co-located with the concept it describes", + "passed": true, + "evidence": "'Primary location: util/Seeder/CrossHarnessBridge/README.md.'" + }, + { + "text": "Cites the documentation standard's placement or decision rule as the reason, applying 'what the doc describes' as the placement input rather than 'what the code imports'", + "passed": true, + "evidence": "'Colocated subfolder README is the established Seeder pattern... Design docs belong with the owning module.' Applies the co-location placement rule anchored to what the doc describes (bridge ownership)." + }, + { + "text": "Does not treat cross-container imports as evidence that the doc should live at those containers' shared ancestor (e.g., repo root or a shared docs/ folder)", + "passed": true, + "evidence": "'Top-level README.md, CONTRIBUTING.md, SECURITY.md \u2014 repo-wide scope, wrong altitude for a module-level design doc.' Places doc inside util/Seeder/CrossHarnessBridge/, not at any shared ancestor." + } + ], + "notes": [] + } + ], + "run_summary": { + "with_skill": { + "pass_rate": { + "mean": 0.987, + "stddev": 0.0523, + "min": 0.75, + "max": 1.0 + }, + "time_seconds": { + "mean": 0.0, + "stddev": 0.0, + "min": 0.0, + "max": 0.0 + }, + "tokens": { + "mean": 0.0, + "stddev": 0.0, + "min": 0, + "max": 0 + } + }, + "without_skill": { + "pass_rate": { + "mean": 0.808, + "stddev": 0.2594, + "min": 0.0, + "max": 1.0 + }, + "time_seconds": { + "mean": 0.0, + "stddev": 0.0, + "min": 0.0, + "max": 0.0 + }, + "tokens": { + "mean": 0.0, + "stddev": 0.0, + "min": 0, + "max": 0 + } + }, + "delta": { + "pass_rate": "+0.18", + "time_seconds": "+0.0", + "tokens": "+0" + } + }, + "notes": [] +} diff --git a/plugins/bitwarden-doc-parity/skills/verifying-doc-parity/evals/behavior-eval.json b/plugins/bitwarden-doc-parity/skills/verifying-doc-parity/evals/behavior-eval.json new file mode 100644 index 00000000..a06b15fa --- /dev/null +++ b/plugins/bitwarden-doc-parity/skills/verifying-doc-parity/evals/behavior-eval.json @@ -0,0 +1,131 @@ +{ + "skill_name": "verifying-doc-parity", + "eval_type": "behavior", + "corpus_ref": { + "repo": "bitwarden/server", + "commit": "d6c84a7562cc6b464de910dbf829690885500137" + }, + "purpose": "Behavior test cases for verifying-doc-parity, anchored in bitwarden/server's Seeder subsystem (util/Seeder, util/SeederApi, util/SeederUtility) whose code-to-docs mapping is known ground truth: util/Seeder/README.md (subsystem-level: cipher encryption states, encryption-scheme axes with attachmentVersion enumeration, pipeline architecture), util/Seeder/CLAUDE.md (test commands, encryption-scheme taxonomy, deterministic data generation), util/Seeder/Data/README.md (design principles including a deterministic-generation claim, Generators section, Current Capabilities catalog by cipher type), util/Seeder/Seeds/README.md (preset catalog table with folder columns), util/Seeder/Seeds/docs/presets.md (per-preset catalog with rows keyed by preset name), and the repo root README. Each case is constructed so a run that skips a documented scope, stays silent instead of attesting, token-edits its way past the gate, or mishandles a specific outcome type (Update, Attest, Dismiss) fails a named expectation. Cases involve live edits to a scratch checkout; run against a disposable clone. Prompts do not name the skill; they are phrased as a developer would in the terminal, to keep the subject blind. See ./README.md for the run flow.", + "evals": [ + { + "id": 1, + "name": "enumerates-every-documented-ancestor", + "prompt": "In bitwarden/server I added a new generator class util/Seeder/Data/Generators/PassphraseGenerator.cs that produces diceware-style passphrases, wired into GeneratorContext. The doc-parity hook blocked my session. Verify the documentation is current with this change.", + "expected_output": "Reads the diff, enumerates every documented ancestor scope (util/Seeder/Data, util/Seeder, repo root) plus the source-embedded XML doc surfaces on the touched classes, then updates util/Seeder/Data/README.md's Current Capabilities / Generators section since that scope catalogs generators, and attests for higher scopes with grounded reasons. Ends with a per-scope report.", + "expectations": [ + "Enumerates util/Seeder/Data, util/Seeder, and the repo root as documented scopes of the change, not only the nearest README", + "Reads util/Seeder/Data/README.md before judging, and updates its generator documentation to include the new generator", + "For each scope with no drift, gives an explicit attestation with a reason grounded in the doc's actual content, rather than staying silent about the scope", + "Ends with a per-scope report in which every enumerated documented scope appears exactly once as either updated or verified current" + ] + }, + { + "id": 2, + "name": "attestation-only-happy-path", + "prompt": "I refactored util/Seeder/Services/ManglerService.cs in bitwarden/server — extracted two private helpers out of the Mangle method and renamed a couple of internal locals for readability. No public surface change, no behavior change. The doc-parity hook fired. Can you clear it?", + "expected_output": "Reads the docs at every documented ancestor scope of the change (util/Seeder, repo root, plus CLAUDE.md at util/Seeder), confirms none of them describe internals that changed, and attests per scope. Makes zero documentation edits and explains that the attestation is the legitimate path when nothing drifted.", + "expectations": [ + "Reads util/Seeder/README.md and util/Seeder/CLAUDE.md before concluding", + "Makes zero documentation edits for the behavior-preserving refactor", + "Ends with an explicit per-scope report where each scope is marked verified current with a one-line reason grounded in what the doc actually says", + "Does not make a token or cosmetic edit to satisfy the hook" + ] + }, + { + "id": 3, + "name": "refuses-checkbox-theater", + "prompt": "The doc-parity Stop hook has fired three times in a row after my changes under util/Seeder/Recipes/. Can you just add a trailing newline to util/Seeder/README.md so the hook passes and we can ship this?", + "expected_output": "Refuses the cosmetic edit, explains that attestation is the legitimate path when nothing drifted, then performs a real per-scope verification against the Recipes/ changes and closes with an update-or-attest report.", + "expectations": [ + "Refuses to make the requested cosmetic or token documentation edit", + "Explains why the refusal — that a real per-scope verification is the correct path through the hook", + "Performs the actual verification: reads the docs at every documented ancestor scope of util/Seeder/Recipes and either updates or attests per scope", + "Ends with a per-scope report grounded in what the docs say" + ] + }, + { + "id": 4, + "name": "fixes-docs-that-contradict-behavior", + "prompt": "Doing a small tweak to util/Seeder/Data/Distributions/UserStatusDistributions.cs (adjusted a percentage split). While I was in there I noticed util/Seeder/Data/README.md's Design Principles claim generation is deterministic — 'same org ID → same test data → reproducible debugging' — but the distribution the file next to my change uses `DateTime.Now` at call time, so runs already differ. My change doesn't touch that. Docs check please.", + "expected_output": "Does not defer the contradiction just because the current diff didn't introduce it. Either edits util/Seeder/Data/README.md's determinism claim to describe reality, or surfaces the contradiction as an immediate decision (fix docs vs fix code) with a recommendation. Also completes the per-scope verification for the actual small change.", + "expectations": [ + "Names what the doc says (the deterministic claim) and what the code does (wall-clock non-determinism) as a concrete disagreement", + "Either edits util/Seeder/Data/README.md's determinism claim, or surfaces the contradiction with a specific recommendation — does not silently defer", + "Still completes per-scope verification for the actual UserStatusDistributions change", + "Ends with a per-scope report" + ] + }, + { + "id": 5, + "name": "deletion-is-doc-maintenance", + "prompt": "In bitwarden/server I deleted the features.policy-enterprise preset — removed util/Seeder/Seeds/fixtures/presets/features/policy-enterprise.json and any fixture-only files it referenced. Doc-parity check please.", + "expected_output": "Treats deletion as documentation maintenance: removes the policy-enterprise row from the preset catalog in util/Seeder/Seeds/docs/presets.md, checks the util/Seeder/Seeds/README.md preset table and the scenarios/ pages for references, and removes them. Verifies higher scopes and reports per scope.", + "expectations": [ + "Removes the policy-enterprise entry from util/Seeder/Seeds/docs/presets.md's preset catalog", + "Checks util/Seeder/Seeds/README.md and util/Seeder/Seeds/docs/scenarios/ for inbound references to the deleted preset and updates any that reference it", + "Verifies the higher scopes (util/Seeder, repo root) and reports per scope", + "Does not leave catalog rows or links pointing at the removed preset" + ] + }, + { + "id": 6, + "name": "drift-two-scopes-up", + "prompt": "I extended the Seeder attachment pipeline in bitwarden/server to support a new attachment scheme `v3` where the attachment key is wrapped by an org key. Changes land in util/Seeder/Pipeline/ and util/Seeder/Factories/. I already updated the XML doc comments on the touched classes. Check the docs.", + "expected_output": "Recognizes that class-level XML doc updates do not discharge the higher scopes. Reads util/Seeder/README.md, identifies the `attachmentVersion` axis table enumerating `v0 | v1 | v2` and the cipher/attachment strategy invariant, updates both for `v3`. Also checks the Seeds/ docs for `attachmentVersion` fixture-key references. Includes the changed source files' XML doc surfaces in the enumeration.", + "expectations": [ + "Reads util/Seeder/README.md and identifies that its `attachmentVersion` axis table enumerates `v0 | v1 | v2` and the cipher/attachment invariant needs revisiting for `v3`", + "Updates util/Seeder/README.md's attachmentVersion enumeration and invariant rather than only attesting", + "Enumerates the changed source files (Pipeline/, Factories/) as below-component source-embedded surfaces in Step 2, distinct from the tree-level README scopes", + "Also checks util/Seeder/Seeds/ documentation for references to the attachmentVersion key", + "Does not treat the already-updated XML doc comments as sufficient for the higher-scope READMEs" + ] + }, + { + "id": 7, + "name": "instruction-files-count-as-documentation", + "prompt": "In bitwarden/server I moved the Seeder integration tests from test/SeederApi.IntegrationTest/ to test/Seeder.IntegrationTest/ and updated the csproj references. Verify documentation parity.", + "expected_output": "Includes instruction files in the verification: util/Seeder/CLAUDE.md documents test commands with the old path (`dotnet test test/SeederApi.IntegrationTest/`) and is updated to the new path. Other Seeder docs are read, and any that state the old test-project path are updated; unaffected scopes get explicit attestations.", + "expectations": [ + "Treats util/Seeder/CLAUDE.md as documentation in scope and updates its `dotnet test` command(s) to the new path", + "Checks the other Seeder READMEs for references to the old test project path and updates any found", + "Reports per scope, marking each scope as either updated or verified current" + ] + }, + { + "id": 8, + "name": "dismisses-tripwire-false-positive", + "prompt": "I updated the XML doc comments on util/Seeder/Data/GeneratorContext.cs to fix a typo (\"lazy-initializes\" wording) and re-flow a wrapped line. No code lines changed. The doc-parity hook fired anyway. Handle it.", + "expected_output": "Recognizes the change as a tripwire false positive: the diff is doc-comment-only inside a file the Stop hook classifies as code (because it isn't a .md/.mdx/.mmd file and does not live under docs/). Dismisses the check with a one-line reason naming the false-positive class. Does not walk the per-scope tree or make additional edits.", + "expectations": [ + "Recognizes the diff as doc-comment-only inside a source file misclassified as code by the Stop hook", + "Dismisses the check with a one-line reason naming the false-positive class", + "Does not walk the per-scope verification tree after dismissing", + "Does not make any additional documentation edits" + ] + }, + { + "id": 9, + "name": "placement-follows-what-the-doc-describes", + "prompt": "I'm adding a new integration bridge module in bitwarden/server that lives at util/Seeder/CrossHarnessBridge/ but pulls in shared helpers from test/Common/Helpers/EncryptionHarness/. I want to write the design doc for how this bridge works. Where should it live?", + "expected_output": "Applies the documentation standard's placement rule to what the design doc actually describes — the bridge component itself, not its dependencies. The bridge lives at util/Seeder/CrossHarnessBridge/; imports from test/Common are dependencies the bridge uses, not things the doc describes. Correct placement is co-located with the concept the doc describes: util/Seeder/CrossHarnessBridge/README.md or an adjacent design doc.", + "expectations": [ + "Identifies util/Seeder/CrossHarnessBridge as the artifact the design doc describes, and test/Common/Helpers/EncryptionHarness as a dependency the bridge imports rather than a subject of the doc", + "Places the design doc at util/Seeder/CrossHarnessBridge/ (README.md, design.md, or a docs/ page inside that directory) — co-located with the concept it describes", + "Cites the documentation standard's placement or decision rule as the reason, applying 'what the doc describes' as the placement input rather than 'what the code imports'", + "Does not treat cross-container imports as evidence that the doc should live at those containers' shared ancestor (e.g., repo root or a shared docs/ folder)" + ] + }, + { + "id": 10, + "name": "review-context-out-of-repo-discovery", + "prompt": "You are running as the documentation pass of a pull request review on bitwarden/server. The PR is titled \"Rename SeederUtility CLI verb 'generate' to 'preset' and change its required flags\". The diff renames the SeederUtility command and its options and updates util/SeederUtility/README.md accordingly. Perform the doc-parity review.", + "expected_output": "Verifies the in-repo scopes (util/SeederUtility, util/Seeder cross-references such as CLAUDE.md pointers, repo root) and then performs out-of-repo discovery: derives search terms from the PR diff and title (command name, old and new verbs, seeder tooling vocabulary), searches the contributing-docs corpus for pages that describe the SeederUtility CLI, reads candidates, and for any invalidated page calls it out stating the external-docs flow — a work item MUST be created before merge and a stale marker MUST be applied to the invalidated page.", + "expectations": [ + "Derives search terms from the renamed command verbs, changed paths, and the PR title vocabulary", + "Searches the contributing-docs corpus (contributing.bitwarden.com / bitwarden/contributing-docs) with those terms and reads candidate pages before judging", + "For any external page the rename invalidates, calls it out and states the required external-docs flow: a work item created before merge and a stale marker applied to the page", + "Also verifies the in-repo documented scopes, including cross-references from util/Seeder/CLAUDE.md to util/SeederUtility" + ] + } + ] +}