From 69f6a972e1f2fd7c9a267b43480c9c1924875f3f Mon Sep 17 00:00:00 2001 From: Dennis Winter Date: Sat, 18 Jul 2026 13:59:20 +0200 Subject: [PATCH 1/8] Add model delegation workflows --- .claude/agents/committer.md | 25 ++++++++++++++++ .codex/agents/committer.toml | 21 +++++++++++++ AGENTS.md | 57 +++++++++++++++++++++++++++++------- CLAUDE.md | 56 +++++++++++++++++++++++++++++++++++ 4 files changed, 148 insertions(+), 11 deletions(-) create mode 100644 .claude/agents/committer.md create mode 100644 .codex/agents/committer.toml diff --git a/.claude/agents/committer.md b/.claude/agents/committer.md new file mode 100644 index 0000000..be851e6 --- /dev/null +++ b/.claude/agents/committer.md @@ -0,0 +1,25 @@ +--- +name: committer +description: Stages a named file list and commits it with a commit message pre-generated by the lead agent. Mechanical execution only — never writes or rewords the message itself. +tools: Bash, Read, Grep +model: haiku +--- + +You execute git commits in this repository. The lead agent's delegation +prompt provides two inputs: the exact commit message and the list of files +to stage. Both are final — you execute, you do not author. + +1. If either input is missing, stop and report what's missing. Do not + invent a message or guess the file list. +2. Cross-check with `git status`: every listed file must actually have + changes, and note any changed files *not* on the list. On mismatch, + stop and report the difference instead of improvising. +3. Stage exactly the listed files by path. Never `git add -A` or + `git add .`. +4. Commit with the provided message verbatim — no rewording, no additions. + Only exception: if it lacks a `Co-Authored-By:` trailer, append + `Co-Authored-By: Claude `. +5. This repo runs pre-commit hooks — if a hook fails, report the hook + output verbatim and stop. Never use `--no-verify` and never edit files + to "fix" a hook failure; that is the lead agent's call. +6. Report the resulting commit hash. Do not push. diff --git a/.codex/agents/committer.toml b/.codex/agents/committer.toml new file mode 100644 index 0000000..b916ee9 --- /dev/null +++ b/.codex/agents/committer.toml @@ -0,0 +1,21 @@ +name = "committer" +description = """ +Stages an explicit file list and commits it with a message authored by the +lead agent. Mechanical execution only; never invent scope or wording. +""" +model = "gpt-5.6-terra" +model_reasoning_effort = "low" +sandbox_mode = "workspace-write" +developer_instructions = """ +Execute git commits in this repository from two final inputs supplied by the +lead agent: an exact commit message and an exact list of files to stage. + +1. If either input is missing, stop and report what is missing. +2. Cross-check the file list against `git status`. Every listed file must have + changes. Report changed files outside the list, and stop on any mismatch. +3. Stage exactly the listed paths. Never use `git add -A` or `git add .`. +4. Commit with the supplied message verbatim. Do not reword it or add trailers. +5. Never use `--no-verify`. If a hook fails, report its output verbatim, make no + fixes, and stop. +6. Report the resulting commit hash. Do not push. +""" diff --git a/AGENTS.md b/AGENTS.md index 8b00564..5e8feef 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,17 +16,18 @@ PRs, full at the merge gate. ## Mechanism map -| Mechanism | Where | Doc | -| ------------------- | -------------------------------- | -------------------------------------------------------------------- | -| Orchestration | `.pre-commit-config.yaml` (root) | this file + inline comments | -| Configs | `.config/` | [`.config/README.md`](.config/README.md) | -| Linting & testing | `.config/scripts/` | [`.config/scripts/README.md`](.config/scripts/README.md) | -| CI & automation | `.github/` | [`.github/README.md`](.github/README.md) | -| Editor integration | `.vscode/` | [`.vscode/README.md`](.vscode/README.md) | -| Policy rules | `.config/PSScriptAnalyzerRules/` | [`.config/scripts/README.md`](.config/scripts/README.md) | -| Opt-in tooling | `.config/overlays/` | [`.config/overlays/vale/README.md`](.config/overlays/vale/README.md) | -| One-time repo setup | `setup/` (delete after use) | [`setup/README.md`](setup/README.md) | -| Optional AI skills | `setup/optional-skills/` | [`setup/README.md`](setup/README.md) | +| Mechanism | Where | Doc | +| ------------------- | ----------------------------------- | -------------------------------------------------------------------- | +| Orchestration | `.pre-commit-config.yaml` (root) | this file + inline comments | +| Configs | `.config/` | [`.config/README.md`](.config/README.md) | +| Linting & testing | `.config/scripts/` | [`.config/scripts/README.md`](.config/scripts/README.md) | +| CI & automation | `.github/` | [`.github/README.md`](.github/README.md) | +| Editor integration | `.vscode/` | [`.vscode/README.md`](.vscode/README.md) | +| Policy rules | `.config/PSScriptAnalyzerRules/` | [`.config/scripts/README.md`](.config/scripts/README.md) | +| Opt-in tooling | `.config/overlays/` | [`.config/overlays/vale/README.md`](.config/overlays/vale/README.md) | +| AI delegation | `.claude/agents/`, `.codex/agents/` | this file + [`CLAUDE.md`](CLAUDE.md) | +| One-time repo setup | `setup/` (delete after use) | [`setup/README.md`](setup/README.md) | +| Optional AI skills | `setup/optional-skills/` | [`setup/README.md`](setup/README.md) | Root convention files are declarative and self-documenting: `.editorconfig` (style), `.gitattributes` (eol=lf), `.gitignore`, `.claudeignore`. @@ -57,6 +58,40 @@ in place. The bundle is deleted with the rest of `setup/` at the end of bootstrap — install it *before* running `Complete-Setup.ps1` if you want to keep it. +## Model delegation (Codex) + +Keep the lead agent responsible for requirements, architecture, debugging, +review conclusions, and changes to hook or policy configuration. Use Codex +subagents only for bounded work that benefits from isolation or can run +independently; subagents consume extra tokens, so do not delegate a tiny +sequential task merely to use a cheaper model. + +Delegate these named task types: + +- **Git commits** → `committer` custom agent (defined in + `.codex/agents/committer.toml`). The lead supplies the final commit message + and exact file list; the subagent only validates, stages, and commits. +- **Codebase recon, log scans, test-failure triage, and dependency scans** → + `explorer` or a read-focused subagent using the fast, lower-cost model. + Return conclusions and relevant file references, not raw dumps. +- **Independent lint or test lanes** → one subagent per lane when they can run + concurrently. Report failures verbatim and fix nothing. +- **Web recon and research summaries** → a read-focused subagent using the + fast model; use the lead model when conclusions require cross-source + judgment. +- **Boilerplate, doc drafts, and PR-description drafts** → a worker subagent + using the fast model and a lead-provided outline or finished facts. The lead + reviews the result and retains responsibility for correctness. +- **Mechanical GitHub chores** → a worker subagent using lead-authored text. + Do not delegate decisions about scope, labels, reviewers, or readiness. + +Prefer parallel subagents for independent read-heavy work. Serialize +write-heavy tasks that touch the same files. Never delegate cross-file +refactors, final review judgment, or edits to `.pre-commit-config.yaml` and +policy rules. When a recurring task needs stricter instructions or a pinned +model, add a project-scoped `.codex/agents/.toml`; keep one-off +delegation in the lead's prompt. + ## Reuse: prefer an include over a copy; a symlink only when no include exists When the same content must exist in more than one place, in order of diff --git a/CLAUDE.md b/CLAUDE.md index 0a685fb..763baaf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,3 +5,59 @@ This repo's tooling architecture, conventions, and retrofit guide live in defines the TDD workflow and where the bundled skill lives. @AGENTS.md + +## Model delegation (Claude Code) + +Reserve the lead model (Fable/Opus) for reasoning-heavy work: architecture, +debugging, code review, and anything touching `.pre-commit-config.yaml` +semantics or policy rules. Delegate the following **named task types** to +cheaper models via the Task tool instead of doing them inline: + +- **Git commits** → `committer` subagent (haiku, defined in + `.claude/agents/committer.md`). The lead model writes the commit message + and passes it plus the explicit file list in the delegation prompt; the + subagent only stages and commits. +- **PR descriptions / `gh pr create`** → general-purpose subagent, + `model: sonnet`. Must fill every section of + `.github/pull_request_template.md`, including Evidence. +- **Log, CI-output, and test-failure scans** → Explore or general-purpose + subagent, `model: haiku`. Return conclusions, not dumps. +- **Boilerplate** (README stubs, config skeletons, test scaffolds) → + general-purpose subagent, `model: sonnet`. +- **Web recon / research summaries** → general-purpose subagent, + `model: haiku` (sonnet when synthesis across sources is needed). +- **Lint/test runs and triage** (`pre-commit run --all-files`, test + suites) → general-purpose subagent, `model: haiku`. Report failures + verbatim; fix nothing. +- **`gh` chores** (file issues, set labels, post comments) → + general-purpose subagent, `model: haiku`, executing lead-authored text. +- **Doc drafts** (AsciiDoc/README skeletons from a lead-provided outline) + → general-purpose subagent, `model: sonnet`. +- **Codebase recon** ("where is X handled?") → Explore subagent, + `model: haiku`. +- **Dependency/changelog scans** → general-purpose subagent, + `model: haiku`. + +In every case the lead model authors, the delegate executes: pass finished +text, outlines, or file lists in the delegation prompt — never delegate the +judgment itself. Delegate only these named task types — not by a vague +"this looks simple" judgment. Never delegate cross-file refactors or edits +to hook/policy configuration. + +### Creating a new delegate mid-task + +When a delegable pattern emerges that the list above doesn't cover: + +1. **One-off** → no file. Spawn a general-purpose or Explore subagent with + the `model:` parameter and a tight, self-contained prompt. Works + immediately in any session. +2. **Recurring** → write `.claude/agents/.md` (frontmatter: `name`, + `description`, `tools`, `model`; body follows the lead-authors, + delegate-executes pattern) and add its task type to the list above. + Claude Code hot-reloads agent files within seconds, so it is invocable + from the next delegation — no restart. +3. **Fallback — restart required** only if `.claude/agents/` did not exist + at session start (first agent in a fresh repo) or the session runs with + `--disable-slash-commands`: write the agent file anyway, write a + handoff (decisions made, open items, next step), then stop and tell the + user to start a new session and send "continue". From fddb73f930e61ce290b2f9b5c34b911bb1d7c1d4 Mon Sep 17 00:00:00 2001 From: Dennis Winter Date: Sat, 18 Jul 2026 14:14:25 +0200 Subject: [PATCH 2/8] Avoid Gitleaks false positive in key fixture --- setup/tests/New-AIMaintainerApp.Tests.ps1 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/setup/tests/New-AIMaintainerApp.Tests.ps1 b/setup/tests/New-AIMaintainerApp.Tests.ps1 index 220bd3e..616ada9 100644 --- a/setup/tests/New-AIMaintainerApp.Tests.ps1 +++ b/setup/tests/New-AIMaintainerApp.Tests.ps1 @@ -60,9 +60,14 @@ Describe 'Get-ManifestFormHtml' -Tag 'Fast' { } Describe 'Save-AppCredential' -Tag 'Fast' { + BeforeAll { + $keyType = 'RSA PRIVATE KEY' + $script:FakePrivateKey = "-----BEGIN $keyType-----`nkey`n-----END $keyType-----" + } + It 'writes the private key next to nothing else and returns its path' { $dir = Join-Path $TestDrive 'keys' - $app = @{ id = 42; slug = 'my-bot'; pem = "-----BEGIN RSA PRIVATE KEY-----`nkey`n-----END RSA PRIVATE KEY-----" } + $app = @{ id = 42; slug = 'my-bot'; pem = $script:FakePrivateKey } $path = Save-AppCredential -App $app -OutputDirectory $dir $path | Should -Be (Join-Path $dir 'my-bot.private-key.pem') Get-Content $path -Raw | Should -Match 'BEGIN RSA PRIVATE KEY' @@ -71,7 +76,7 @@ Describe 'Save-AppCredential' -Tag 'Fast' { It 'hardens private-key permissions immediately after writing' { Mock Protect-PrivateKeyFile {} $dir = Join-Path $TestDrive 'keys' - $app = @{ id = 42; slug = 'my-bot'; pem = "-----BEGIN RSA PRIVATE KEY-----`nkey`n-----END RSA PRIVATE KEY-----" } + $app = @{ id = 42; slug = 'my-bot'; pem = $script:FakePrivateKey } $path = Save-AppCredential -App $app -OutputDirectory $dir From 4a4409f0c956bc3d1655b9ea0d43866f5178e404 Mon Sep 17 00:00:00 2001 From: Dennis Winter Date: Sat, 18 Jul 2026 14:22:14 +0200 Subject: [PATCH 3/8] Align Gitleaks versions across local and CI --- .github/README.md | 5 +++-- .github/workflows/lint.yml | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/README.md b/.github/README.md index e64deec..3500f40 100644 --- a/.github/README.md +++ b/.github/README.md @@ -27,8 +27,9 @@ This is the one CI check that is *not* a local pre-commit hook, by design: the pre-commit gitleaks hook scans only staged changes (a no-op in CI) and local hooks are bypassable, so this server-side scan is the unbypassable backstop over the whole history / PR range. It runs the same gitleaks engine, so no lint logic -is duplicated. Org-owned repos need a free `GITLEAKS_LICENSE` secret; user-owned -repos do not. +is duplicated. `GITLEAKS_VERSION` in the workflow is kept in sync with the +managed hook revision so local and CI findings agree. Org-owned repos need a +free `GITLEAKS_LICENSE` secret; user-owned repos do not. ### Protecting the default branch (once per repo) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a3ca703..46feb4a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -46,6 +46,8 @@ jobs: uses: gitleaks/gitleaks-action@e0c47f4f8be36e29cdc102c57e68cb5cbf0e8d1e # v3.0.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Keep aligned with the managed hook in .pre-commit-config.yaml. + GITLEAKS_VERSION: 8.30.0 # Org accounts need a free GITLEAKS_LICENSE here; user-owned repos do not. - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 From 1ca546f232cf1b319d97c8841d6a1aa943af5204 Mon Sep 17 00:00:00 2001 From: Dennis Winter Date: Sat, 18 Jul 2026 14:38:05 +0200 Subject: [PATCH 4/8] Clarify Gitleaks scan consistency. --- .github/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/README.md b/.github/README.md index 3500f40..767cb09 100644 --- a/.github/README.md +++ b/.github/README.md @@ -28,8 +28,9 @@ pre-commit gitleaks hook scans only staged changes (a no-op in CI) and local hooks are bypassable, so this server-side scan is the unbypassable backstop over the whole history / PR range. It runs the same gitleaks engine, so no lint logic is duplicated. `GITLEAKS_VERSION` in the workflow is kept in sync with the -managed hook revision so local and CI findings agree. Org-owned repos need a -free `GITLEAKS_LICENSE` secret; user-owned repos do not. +managed hook revision so rule definitions do not drift between local and CI +scans. Org-owned repos need a free `GITLEAKS_LICENSE` secret; user-owned repos +do not. ### Protecting the default branch (once per repo) From 1cd3262454d6ffbd89a35e353b84f74bd460d6c7 Mon Sep 17 00:00:00 2001 From: Dennis Winter Date: Sat, 18 Jul 2026 14:44:05 +0200 Subject: [PATCH 5/8] Isolate Codex delegation policy --- .codex/config.toml | 39 +++++++++++++++++++++++++++++++ AGENTS.md | 58 ++++++++++------------------------------------ 2 files changed, 51 insertions(+), 46 deletions(-) create mode 100644 .codex/config.toml diff --git a/.codex/config.toml b/.codex/config.toml new file mode 100644 index 0000000..aaed670 --- /dev/null +++ b/.codex/config.toml @@ -0,0 +1,39 @@ +#:schema https://developers.openai.com/codex/config-schema.json + +# Codex reads AGENTS.md directly. Keep only Codex-specific additions here so +# other agents can reuse AGENTS.md without inheriting this delegation policy. +developer_instructions = """ +## Model delegation (Codex) + +Keep the lead agent responsible for requirements, architecture, debugging, +review conclusions, and changes to hook or policy configuration. Use Codex +subagents only for bounded work that benefits from isolation or can run +independently; subagents consume extra tokens, so do not delegate a tiny +sequential task merely to use a cheaper model. + +Delegate these named task types: + +- **Git commits** → `committer` custom agent (defined in + `.codex/agents/committer.toml`). The lead supplies the final commit message + and exact file list; the subagent only validates, stages, and commits. +- **Codebase recon, log scans, test-failure triage, and dependency scans** → + `explorer` or a read-focused subagent using the fast, lower-cost model. + Return conclusions and relevant file references, not raw dumps. +- **Independent lint or test lanes** → one subagent per lane when they can run + concurrently. Report failures verbatim and fix nothing. +- **Web recon and research summaries** → a read-focused subagent using the + fast model; use the lead model when conclusions require cross-source + judgment. +- **Boilerplate, doc drafts, and PR-description drafts** → a worker subagent + using the fast model and a lead-provided outline or finished facts. The lead + reviews the result and retains responsibility for correctness. +- **Mechanical GitHub chores** → a worker subagent using lead-authored text. + Do not delegate decisions about scope, labels, reviewers, or readiness. + +Prefer parallel subagents for independent read-heavy work. Serialize +write-heavy tasks that touch the same files. Never delegate cross-file +refactors, final review judgment, or edits to `.pre-commit-config.yaml` and +policy rules. When a recurring task needs stricter instructions or a pinned +model, add a project-scoped `.codex/agents/.toml`; keep one-off +delegation in the lead's prompt. +""" diff --git a/AGENTS.md b/AGENTS.md index 5e8feef..f39adb1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,18 +16,18 @@ PRs, full at the merge gate. ## Mechanism map -| Mechanism | Where | Doc | -| ------------------- | ----------------------------------- | -------------------------------------------------------------------- | -| Orchestration | `.pre-commit-config.yaml` (root) | this file + inline comments | -| Configs | `.config/` | [`.config/README.md`](.config/README.md) | -| Linting & testing | `.config/scripts/` | [`.config/scripts/README.md`](.config/scripts/README.md) | -| CI & automation | `.github/` | [`.github/README.md`](.github/README.md) | -| Editor integration | `.vscode/` | [`.vscode/README.md`](.vscode/README.md) | -| Policy rules | `.config/PSScriptAnalyzerRules/` | [`.config/scripts/README.md`](.config/scripts/README.md) | -| Opt-in tooling | `.config/overlays/` | [`.config/overlays/vale/README.md`](.config/overlays/vale/README.md) | -| AI delegation | `.claude/agents/`, `.codex/agents/` | this file + [`CLAUDE.md`](CLAUDE.md) | -| One-time repo setup | `setup/` (delete after use) | [`setup/README.md`](setup/README.md) | -| Optional AI skills | `setup/optional-skills/` | [`setup/README.md`](setup/README.md) | +| Mechanism | Where | Doc | +| ------------------- | -------------------------------- | -------------------------------------------------------------------- | +| Orchestration | `.pre-commit-config.yaml` (root) | this file + inline comments | +| Configs | `.config/` | [`.config/README.md`](.config/README.md) | +| Linting & testing | `.config/scripts/` | [`.config/scripts/README.md`](.config/scripts/README.md) | +| CI & automation | `.github/` | [`.github/README.md`](.github/README.md) | +| Editor integration | `.vscode/` | [`.vscode/README.md`](.vscode/README.md) | +| Policy rules | `.config/PSScriptAnalyzerRules/` | [`.config/scripts/README.md`](.config/scripts/README.md) | +| Opt-in tooling | `.config/overlays/` | [`.config/overlays/vale/README.md`](.config/overlays/vale/README.md) | +| AI delegation | `.claude/`, `.codex/` | [`CLAUDE.md`](CLAUDE.md) + tool-specific config | +| One-time repo setup | `setup/` (delete after use) | [`setup/README.md`](setup/README.md) | +| Optional AI skills | `setup/optional-skills/` | [`setup/README.md`](setup/README.md) | Root convention files are declarative and self-documenting: `.editorconfig` (style), `.gitattributes` (eol=lf), `.gitignore`, `.claudeignore`. @@ -58,40 +58,6 @@ in place. The bundle is deleted with the rest of `setup/` at the end of bootstrap — install it *before* running `Complete-Setup.ps1` if you want to keep it. -## Model delegation (Codex) - -Keep the lead agent responsible for requirements, architecture, debugging, -review conclusions, and changes to hook or policy configuration. Use Codex -subagents only for bounded work that benefits from isolation or can run -independently; subagents consume extra tokens, so do not delegate a tiny -sequential task merely to use a cheaper model. - -Delegate these named task types: - -- **Git commits** → `committer` custom agent (defined in - `.codex/agents/committer.toml`). The lead supplies the final commit message - and exact file list; the subagent only validates, stages, and commits. -- **Codebase recon, log scans, test-failure triage, and dependency scans** → - `explorer` or a read-focused subagent using the fast, lower-cost model. - Return conclusions and relevant file references, not raw dumps. -- **Independent lint or test lanes** → one subagent per lane when they can run - concurrently. Report failures verbatim and fix nothing. -- **Web recon and research summaries** → a read-focused subagent using the - fast model; use the lead model when conclusions require cross-source - judgment. -- **Boilerplate, doc drafts, and PR-description drafts** → a worker subagent - using the fast model and a lead-provided outline or finished facts. The lead - reviews the result and retains responsibility for correctness. -- **Mechanical GitHub chores** → a worker subagent using lead-authored text. - Do not delegate decisions about scope, labels, reviewers, or readiness. - -Prefer parallel subagents for independent read-heavy work. Serialize -write-heavy tasks that touch the same files. Never delegate cross-file -refactors, final review judgment, or edits to `.pre-commit-config.yaml` and -policy rules. When a recurring task needs stricter instructions or a pinned -model, add a project-scoped `.codex/agents/.toml`; keep one-off -delegation in the lead's prompt. - ## Reuse: prefer an include over a copy; a symlink only when no include exists When the same content must exist in more than one place, in order of From 47daadce59f5854a4a38724a0143df55d7c329f4 Mon Sep 17 00:00:00 2001 From: Dennis Winter Date: Sat, 18 Jul 2026 15:21:13 +0200 Subject: [PATCH 6/8] Document reduced OKF conventions --- AGENTS.md | 43 ++++++++++++----- docs/knowledge-format.md | 50 ++++++++++++++++++++ setup/MIGRATION.md | 99 +++++++++++++++++++++++++--------------- 3 files changed, 144 insertions(+), 48 deletions(-) create mode 100644 docs/knowledge-format.md diff --git a/AGENTS.md b/AGENTS.md index f39adb1..d87dba6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,18 +16,19 @@ PRs, full at the merge gate. ## Mechanism map -| Mechanism | Where | Doc | -| ------------------- | -------------------------------- | -------------------------------------------------------------------- | -| Orchestration | `.pre-commit-config.yaml` (root) | this file + inline comments | -| Configs | `.config/` | [`.config/README.md`](.config/README.md) | -| Linting & testing | `.config/scripts/` | [`.config/scripts/README.md`](.config/scripts/README.md) | -| CI & automation | `.github/` | [`.github/README.md`](.github/README.md) | -| Editor integration | `.vscode/` | [`.vscode/README.md`](.vscode/README.md) | -| Policy rules | `.config/PSScriptAnalyzerRules/` | [`.config/scripts/README.md`](.config/scripts/README.md) | -| Opt-in tooling | `.config/overlays/` | [`.config/overlays/vale/README.md`](.config/overlays/vale/README.md) | -| AI delegation | `.claude/`, `.codex/` | [`CLAUDE.md`](CLAUDE.md) + tool-specific config | -| One-time repo setup | `setup/` (delete after use) | [`setup/README.md`](setup/README.md) | -| Optional AI skills | `setup/optional-skills/` | [`setup/README.md`](setup/README.md) | +| Mechanism | Where | Doc | +| ------------------- | -------------------------------- | --------------------------------------------------------------------- | +| Orchestration | `.pre-commit-config.yaml` (root) | this file + inline comments | +| Configs | `.config/` | [`.config/README.md`](.config/README.md) | +| Linting & testing | `.config/scripts/` | [`.config/scripts/README.md`](.config/scripts/README.md) | +| CI & automation | `.github/` | [`.github/README.md`](.github/README.md) | +| Editor integration | `.vscode/` | [`.vscode/README.md`](.vscode/README.md) | +| Policy rules | `.config/PSScriptAnalyzerRules/` | [`.config/scripts/README.md`](.config/scripts/README.md) | +| Opt-in tooling | `.config/overlays/` | [`.config/overlays/vale/README.md`](.config/overlays/vale/README.md) | +| Documentation graph | Markdown docs + README indexes | [`docs/knowledge-format.md`](docs/knowledge-format.md) | +| AI delegation | `.claude/`, `.codex/` | [`CLAUDE.md`](CLAUDE.md) + [`.codex/config.toml`](.codex/config.toml) | +| One-time repo setup | `setup/` (delete after use) | [`setup/README.md`](setup/README.md) | +| Optional AI skills | `setup/optional-skills/` | [`setup/README.md`](setup/README.md) | Root convention files are declarative and self-documenting: `.editorconfig` (style), `.gitattributes` (eol=lf), `.gitignore`, `.claudeignore`. @@ -58,6 +59,24 @@ in place. The bundle is deleted with the rest of `setup/` at the end of bootstrap — install it *before* running `Complete-Setup.ps1` if you want to keep it. +## Documentation relationships + +This repo uses a deliberately reduced, non-conformant subset of the draft Open +Knowledge Format: ordinary Markdown links make related documents discoverable, +and README indexes provide progressive disclosure without turning each +consumer repo into a knowledge bundle. The adopted and rejected conventions +are recorded in +[`docs/knowledge-format.md`](docs/knowledge-format.md); do not copy the +upstream draft into this repo. + +**Keep the migration runbook connected.** When a change adds, removes, or +materially changes a permanent mechanism, agent entry point, delegation file, +documentation convention, or setup step, review +[`setup/MIGRATION.md`](setup/MIGRATION.md) in the same change. Update it when +the retrofit steps change; otherwise state in the PR why migration is +unaffected. This rule lives here because Codex reads `AGENTS.md` directly and +Claude imports it through `CLAUDE.md`. + ## Reuse: prefer an include over a copy; a symlink only when no include exists When the same content must exist in more than one place, in order of diff --git a/docs/knowledge-format.md b/docs/knowledge-format.md new file mode 100644 index 0000000..caca108 --- /dev/null +++ b/docs/knowledge-format.md @@ -0,0 +1,50 @@ +# Knowledge format + +## Purpose and status + +This repository uses a deliberately reduced, OKF-inspired convention for +documentation in an ordinary software repository. It is not an LLM wiki or a +portable knowledge bundle, and this policy does not claim Open Knowledge Format +(OKF) conformance or an OKF version. + +OKF is currently a +[version 0.1 draft](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md#open-knowledge-format-okf) +aimed at knowledge that people and agents can author, consume, and exchange. +Treat its upstream specification as the evolving authority; link to it rather +than copying its rules into this repository. + +## Local convention + +Borrow these useful properties: + +- Write human- and agent-readable Markdown that remains easy to review as a Git + diff, reflecting OKF's + [readability and diffability goals](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md#1-motivation). +- Express relationships with explicit, standard Markdown links and enough + surrounding prose to explain the relationship, as in + [OKF cross-linking](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md#5-cross-linking). +- Use a `README.md` as the index for a documentation directory when an index + helps readers discover its contents before opening individual files. This is + the local equivalent of OKF's + [progressive-disclosure index](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md#6-index-files). +- Cite external sources for claims that depend on them, using ordinary Markdown + links near the claim or a compact citations section. OKF likewise treats + [citations as supporting links](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md#8-citations). + +Use ordinary repository-relative links such as `../README.md` or +`./design.md`. Do not use OKF's leading-slash, bundle-relative link convention. + +## Deliberate omissions + +Do not add mandatory YAML frontmatter or a required `type`; those are OKF +[concept-document requirements](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md#41-frontmatter), +not requirements for this repository. Do not reserve `index.md` or `log.md` or +assign them OKF semantics; upstream defines those names specially for +[bundle indexes and update logs](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md#31-reserved-filenames). +Do not describe this repository as an OKF bundle, conformant implementation, or +versioned OKF producer; upstream defines those concepts for +[interoperable bundles](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md#9-conformance). + +Reevaluate this policy if a future consumer needs an exchangeable knowledge +catalog. At that point, assess the then-current upstream specification and adopt +its bundle, metadata, reserved-file, conformance, and versioning rules together. diff --git a/setup/MIGRATION.md b/setup/MIGRATION.md index e871664..3589cd2 100644 --- a/setup/MIGRATION.md +++ b/setup/MIGRATION.md @@ -11,20 +11,22 @@ first. [`CHANGELOG.md`](CHANGELOG.md) has the version history. This file lives in `setup/` alongside the one-time setup tooling — it's only needed during the migration, so it goes away with the rest of the folder when -`Complete-Setup.ps1` runs (step 10). +`Complete-Setup.ps1` runs (step 12). ## The concepts you're migrating -| # | Concept | Lives in | Doc | -| - | ------------------- | -------------------------------- | ------------------------------------------------------------------------ | -| 1 | Orchestration | `.pre-commit-config.yaml` (root) | [`/AGENTS.md`](../AGENTS.md) + inline comments | -| 2 | Configs | `.config/` | [`/.config/README.md`](../.config/README.md) | -| 3 | Linting & testing | `.config/scripts/` | [`/.config/scripts/README.md`](../.config/scripts/README.md) | -| 4 | Policy rules | `.config/PSScriptAnalyzerRules/` | [`/.config/scripts/README.md`](../.config/scripts/README.md) | -| 5 | Opt-in tooling | `.config/overlays/` | [`/.config/overlays/vale/README.md`](../.config/overlays/vale/README.md) | -| 6 | CI & automation | `.github/` | [`/.github/README.md`](../.github/README.md) | -| 7 | Editor integration | `.vscode/` | [`/.vscode/README.md`](../.vscode/README.md) | -| 8 | One-time repo setup | `setup/` (delete after use) | [`README.md`](README.md) | +| # | Concept | Lives in | Doc | +| -- | ------------------- | ---------------------------------- | ------------------------------------------------------------------------ | +| 1 | Orchestration | `.pre-commit-config.yaml` (root) | [`/AGENTS.md`](../AGENTS.md) + inline comments | +| 2 | Configs | `.config/` | [`/.config/README.md`](../.config/README.md) | +| 3 | Linting & testing | `.config/scripts/` | [`/.config/scripts/README.md`](../.config/scripts/README.md) | +| 4 | Policy rules | `.config/PSScriptAnalyzerRules/` | [`/.config/scripts/README.md`](../.config/scripts/README.md) | +| 5 | Opt-in tooling | `.config/overlays/` | [`/.config/overlays/vale/README.md`](../.config/overlays/vale/README.md) | +| 6 | CI & automation | `.github/` | [`/.github/README.md`](../.github/README.md) | +| 7 | Editor integration | `.vscode/` | [`/.vscode/README.md`](../.vscode/README.md) | +| 8 | Documentation graph | Markdown docs + README indexes | [`/docs/knowledge-format.md`](../docs/knowledge-format.md) | +| 9 | AI delegation | `AGENTS.md`, `.claude/`, `.codex/` | [`/CLAUDE.md`](../CLAUDE.md) + tool-specific config | +| 10 | One-time repo setup | `setup/` (delete after use) | [`README.md`](README.md) | Root convention files ride along as-is and need no adaptation: `.editorconfig`, `.gitattributes` (`eol=lf`), `.gitignore`, `.claudeignore`. @@ -56,7 +58,8 @@ repo's root. `repo: local` script (step 4), or be consciously dropped — list the mapping in the migration PR description. *Done when:* `pre-commit run --all-files` executes (failures are fine — - they're the backlog for the next steps). + they're the backlog for the next steps). This is not a repository-history + secret audit: the managed Gitleaks hook intentionally scans staged changes. 3. **Relocate configs** into `.config/` (concept 2), referenced explicitly via hook `args` — keep only root-*forced* files at root. Factor markdown rules @@ -85,7 +88,10 @@ repo's root. 6. **Collapse CI to one workflow** (concept 6) that runs `pre-commit run` — scoped to the diff on `pull_request`, full on `merge_group` and `push: main`. Replace N split per-language workflows; path filtering is no - longer needed since pre-commit scopes by file. Carry over `CODEOWNERS`, + longer needed since pre-commit scopes by file. Keep the separate + full-history/PR-range Gitleaks Action as the deliberate server-side + backstop, and keep its `GITLEAKS_VERSION` aligned with the managed hook + revision so their rule definitions do not drift. Carry over `CODEOWNERS`, issue templates, and `pull_request_template.md` for the same Evidence-mandatory PR contract. *Done when:* exactly one lint workflow remains and a test PR shows the @@ -99,29 +105,50 @@ repo's root. *Done when:* the three files are tracked (`git ls-files .vscode/` lists them) and nothing else in `.vscode/` is. -8. **Copy and run the one-time setup tooling** (concept 8) after the first - push and one CI run (so the `lint` check the ruleset requires actually - exists): copy the whole `setup/` folder (including this file and - `optional-skills/` — install the bundled `develop-with-tdd` skill only if - no equivalent TDD skill exists), then follow - [`README.md`](README.md) — protection, security settings, AI-maintainer - identity, in that order. - **If the target repo is owned by a personal/user account** (not an - organization), expect `merge_queue` to be rejected by GitHub — a platform - limitation, not a bug. `Protect-MainBranch.ps1` posts each rule as its own - ruleset for exactly this reason: the other four protections still land. - *Done when:* the rulesets appear in repo settings and - `Test-AIMaintainerIdentity.ps1` passes from the agent's shell. - -9. **Copy the remaining docs.** `AGENTS.md` plus the per-folder READMEs - (trimmed to the target repo's actual tools), updated to reflect what you - actually migrated. Purge phantoms: verify every referenced tool exists in - the target repo (e.g. there is no `asciidoctor-lint` gem — use - `asciidoctor --failure-level` for syntax and Vale for prose). A doc that - lists a nonexistent tool is worse than no doc. - *Done when:* every tool named in a doc can be invoked in the target repo. - -10. **Finish.** Delete the reference copy (`.temp/template/`), then run +8. **Adopt the documentation graph** (concept 8). Copy + `docs/knowledge-format.md` and adapt its deliberately reduced OKF-inspired + conventions to the target. Keep normal Markdown links between documents, + use README files as directory indexes where progressive disclosure helps, + and cite external claims. Do not add mandatory frontmatter, reserved + `index.md`/`log.md` semantics, or an OKF conformance claim unless the target + is intentionally becoming an exchangeable knowledge bundle. + *Done when:* permanent docs are reachable from `AGENTS.md` or a linked + README and related documents link to each other explicitly. + +9. **Adopt the AI entry points and delegation policy** (concept 9). Copy + `AGENTS.md` as shared repository guidance, then copy only the product + layers the target uses: `CLAUDE.md` and `.claude/agents/` for Claude Code; + `.codex/config.toml` and `.codex/agents/` for Codex. Adapt model names and + supported capabilities rather than copying them blindly. Keep + product-specific policy out of `AGENTS.md`: Claude imports the shared file, + while Codex reads it directly and adds its own `developer_instructions` + from `.codex/config.toml` in trusted projects. + *Done when:* every retained product entry point loads the shared guidance, + every referenced custom agent exists, and no unused product layer remains. + +10. **Copy the remaining docs.** Bring over the per-folder READMEs, trimmed to + the target repo's actual tools, and update them to reflect what you + migrated. Purge phantoms: verify every referenced tool exists in the + target repo (e.g. there is no `asciidoctor-lint` gem — use + `asciidoctor --failure-level` for syntax and Vale for prose). A doc that + lists a nonexistent tool is worse than no doc. + *Done when:* every tool named in a doc can be invoked in the target repo. + +11. **Copy and run the one-time setup tooling** (concept 10) after the first + push and one CI run (so the `lint` check the ruleset requires actually + exists): copy the whole `setup/` folder (including this file and + `optional-skills/` — install the bundled `develop-with-tdd` skill only if + no equivalent TDD skill exists), then follow + [`README.md`](README.md) — protection, security settings, AI-maintainer + identity, in that order. + **If the target repo is owned by a personal/user account** (not an + organization), expect `merge_queue` to be rejected by GitHub — a platform + limitation, not a bug. `Protect-MainBranch.ps1` posts each rule as its own + ruleset for exactly this reason: the other four protections still land. + *Done when:* the rulesets appear in repo settings and + `Test-AIMaintainerIdentity.ps1` passes from the agent's shell. + +12. **Finish.** Delete the reference copy (`.temp/template/`), then run `pwsh -NoProfile -File setup/Complete-Setup.ps1` — it removes `setup/` and opens the removal PR (and refuses to run if the repo is itself a template repo). From aadf62c1c3b5b012039e3c6d446d99d6bb505323 Mon Sep 17 00:00:00 2001 From: Dennis Winter Date: Sat, 18 Jul 2026 17:05:08 +0200 Subject: [PATCH 7/8] Harden setup teardown and expand knowledge docs - add fail-closed AI-maintainer tokens, identity checks, and teardown cleanup - make AsciiDoc first-class and convert the complete human TDD guide - document token-saving trade-offs and narrow committer delegation --- .codex/config.toml | 9 +- .config/README.md | 9 +- .config/overlays/README.md | 17 + .config/overlays/semgrep-pro/README.md | 54 ++- .config/overlays/semgrep/README.md | 30 +- .config/scripts/README.md | 2 + .config/semgrep/README.md | 15 +- .github/CODEOWNERS | 2 +- .github/README.md | 31 +- AGENTS.md | 56 +-- CLAUDE.md | 14 +- README.md | 7 +- docs/README.adoc | 12 + docs/knowledge-format.md | 24 +- docs/token-saving.adoc | 130 +++++++ setup/AI-Maintainer-Identity.adoc | 103 +++-- setup/CHANGELOG.md | 32 ++ setup/Complete-Setup.ps1 | 95 ++++- setup/MIGRATION.md | 83 ++-- setup/New-AIMaintainerApp.ps1 | 3 +- setup/New-AIMaintainerToken.ps1 | 140 +++++++ setup/README.md | 20 +- setup/Test-AIMaintainerIdentity.ps1 | 24 +- setup/optional-skills/README.adoc | 24 ++ .../knowledge/tdd/ASSESSMENT_BRIEF.adoc | 138 +++++++ .../knowledge/tdd/ASSESSMENT_BRIEF.md | 133 ------- .../knowledge/tdd/MAINTENANCE.adoc | 49 +++ .../knowledge/tdd/MAINTENANCE.md | 42 --- .../optional-skills/knowledge/tdd/README.adoc | 84 +++++ setup/optional-skills/knowledge/tdd/README.md | 75 ---- .../knowledge/tdd/analysis.adoc | 226 +++++++++++ .../knowledge/tdd/external-skill-survey.adoc | 283 ++++++++++++++ .../guide/{01-why-tdd.md => 01-why-tdd.adoc} | 106 +++--- ...actising-tdd.md => 02-practising-tdd.adoc} | 194 +++++----- ...mmon-misconceptions-and-failure-modes.adoc | 354 ++++++++++++++++++ ...common-misconceptions-and-failure-modes.md | 339 ----------------- ...choosing-evidence-and-feedback-lanes.adoc} | 100 ++--- ...> 05-bdd-and-collaborative-discovery.adoc} | 103 ++--- ....md => 06-applying-tdd-without-dogma.adoc} | 128 +++---- ...ed-examples.md => 07-worked-examples.adoc} | 260 +++++++------ ...djacent-practices-and-further-reading.adoc | 191 ++++++++++ .../adjacent-practices-and-further-reading.md | 191 ---------- .../appendices/{glossary.md => glossary.adoc} | 150 ++++---- .../appendices/stack-specific-notes.adoc | 55 +++ .../guide/appendices/stack-specific-notes.md | 54 --- .../knowledge/tdd/research-findings.adoc | 171 +++++++++ .../knowledge/tdd/research-findings.md | 164 -------- .../knowledge/tdd/skill-evaluation.adoc | 106 ++++++ .../knowledge/tdd/traceability.adoc | 57 +++ .../knowledge/tdd/traceability.md | 49 --- setup/tests/Complete-Setup.Tests.ps1 | 78 +++- setup/tests/New-AIMaintainerApp.Tests.ps1 | 1 + setup/tests/New-AIMaintainerToken.Tests.ps1 | 83 ++++ .../tests/Test-AIMaintainerIdentity.Tests.ps1 | 34 +- setup/version | 2 +- 55 files changed, 3195 insertions(+), 1741 deletions(-) create mode 100644 .config/overlays/README.md create mode 100644 docs/README.adoc create mode 100644 docs/token-saving.adoc create mode 100644 setup/New-AIMaintainerToken.ps1 create mode 100644 setup/optional-skills/README.adoc create mode 100644 setup/optional-skills/knowledge/tdd/ASSESSMENT_BRIEF.adoc delete mode 100644 setup/optional-skills/knowledge/tdd/ASSESSMENT_BRIEF.md create mode 100644 setup/optional-skills/knowledge/tdd/MAINTENANCE.adoc delete mode 100644 setup/optional-skills/knowledge/tdd/MAINTENANCE.md create mode 100644 setup/optional-skills/knowledge/tdd/README.adoc delete mode 100644 setup/optional-skills/knowledge/tdd/README.md create mode 100644 setup/optional-skills/knowledge/tdd/analysis.adoc create mode 100644 setup/optional-skills/knowledge/tdd/external-skill-survey.adoc rename setup/optional-skills/knowledge/tdd/guide/{01-why-tdd.md => 01-why-tdd.adoc} (74%) rename setup/optional-skills/knowledge/tdd/guide/{02-practising-tdd.md => 02-practising-tdd.adoc} (69%) create mode 100644 setup/optional-skills/knowledge/tdd/guide/03-common-misconceptions-and-failure-modes.adoc delete mode 100644 setup/optional-skills/knowledge/tdd/guide/03-common-misconceptions-and-failure-modes.md rename setup/optional-skills/knowledge/tdd/guide/{04-choosing-evidence-and-feedback-lanes.md => 04-choosing-evidence-and-feedback-lanes.adoc} (64%) rename setup/optional-skills/knowledge/tdd/guide/{05-bdd-and-collaborative-discovery.md => 05-bdd-and-collaborative-discovery.adoc} (61%) rename setup/optional-skills/knowledge/tdd/guide/{06-applying-tdd-without-dogma.md => 06-applying-tdd-without-dogma.adoc} (73%) rename setup/optional-skills/knowledge/tdd/guide/{07-worked-examples.md => 07-worked-examples.adoc} (85%) create mode 100644 setup/optional-skills/knowledge/tdd/guide/appendices/adjacent-practices-and-further-reading.adoc delete mode 100644 setup/optional-skills/knowledge/tdd/guide/appendices/adjacent-practices-and-further-reading.md rename setup/optional-skills/knowledge/tdd/guide/appendices/{glossary.md => glossary.adoc} (79%) create mode 100644 setup/optional-skills/knowledge/tdd/guide/appendices/stack-specific-notes.adoc delete mode 100644 setup/optional-skills/knowledge/tdd/guide/appendices/stack-specific-notes.md create mode 100644 setup/optional-skills/knowledge/tdd/research-findings.adoc delete mode 100644 setup/optional-skills/knowledge/tdd/research-findings.md create mode 100644 setup/optional-skills/knowledge/tdd/skill-evaluation.adoc create mode 100644 setup/optional-skills/knowledge/tdd/traceability.adoc delete mode 100644 setup/optional-skills/knowledge/tdd/traceability.md create mode 100644 setup/tests/New-AIMaintainerToken.Tests.ps1 diff --git a/.codex/config.toml b/.codex/config.toml index aaed670..0db6443 100644 --- a/.codex/config.toml +++ b/.codex/config.toml @@ -13,9 +13,12 @@ sequential task merely to use a cheaper model. Delegate these named task types: -- **Git commits** → `committer` custom agent (defined in - `.codex/agents/committer.toml`). The lead supplies the final commit message - and exact file list; the subagent only validates, stages, and commits. +- **Git commits** are eligible for the `committer` custom agent (defined in + `.codex/agents/committer.toml`) when delegation preserves scarce lead + context, overlaps useful lead work, or the user requests isolation. For a + tiny sequential commit, the lead may commit directly. The lead supplies the + final commit message and exact file list; a delegate only validates, stages, + and commits. See `docs/token-saving.adoc` for the trade-off. - **Codebase recon, log scans, test-failure triage, and dependency scans** → `explorer` or a read-focused subagent using the fast, lower-cost model. Return conclusions and relevant file references, not raw dumps. diff --git a/.config/README.md b/.config/README.md index 0e2537f..8f96cce 100644 --- a/.config/README.md +++ b/.config/README.md @@ -6,7 +6,7 @@ for the overall architecture; the mechanisms documented elsewhere: - Linting & testing scripts → [`scripts/README.md`](scripts/README.md) - CI workflows → [`../.github/README.md`](../.github/README.md) - Editor integration → [`../.vscode/README.md`](../.vscode/README.md) -- Opt-in tooling → [`overlays/`](overlays/) +- Opt-in tooling → [`overlays/README.md`](overlays/README.md) ## Why configs live here @@ -40,10 +40,3 @@ editor and commit stay in agreement. `PSScriptAnalyzerSettings.psd1` gates on `Warning` + `Error`. Suppress specific noisy rules via `ExcludeRules` rather than dropping a whole severity. - -## Overlays - -[`overlays/`](overlays/) is the "add-when-relevant" shelf: complete-but-inert -tooling that never runs until activated (each carries its own config, a -`precommit-hook.yaml` fragment to paste into the root config, and a README). See -[`overlays/vale/`](overlays/vale/). diff --git a/.config/overlays/README.md b/.config/overlays/README.md new file mode 100644 index 0000000..aea590b --- /dev/null +++ b/.config/overlays/README.md @@ -0,0 +1,17 @@ +# Optional tooling overlays + +Overlays are the add-when-relevant shelf: deliberately inert tooling and +decision records that do not run until a maintainer activates them. Read an +overlay's own instructions before copying its hook fragment; not every overlay +has the same shape. + +| Overlay | State | What it contains | +| --------------------------------------- | -------------------- | ----------------------------------------------------- | +| [`vale/`](vale/README.md) | Complete, inert | Vale config, vocabulary, and a pre-commit fragment | +| [`semgrep/`](semgrep/README.md) | Complete, inert | Local-rule activation and a pre-commit fragment | +| [`semgrep-pro/`](semgrep-pro/README.md) | Decision record only | Current product/licensing trade-offs; no active rules | + +Activation is a repository decision: copy or enable only the mechanisms that +match the target repository's languages and risk. Keeping an overlay complete +but inactive avoids broken placeholder checks while preserving a reviewed path +to adoption. diff --git a/.config/overlays/semgrep-pro/README.md b/.config/overlays/semgrep-pro/README.md index 1fd0f7d..0a697a2 100644 --- a/.config/overlays/semgrep-pro/README.md +++ b/.config/overlays/semgrep-pro/README.md @@ -8,41 +8,39 @@ also opt-in, also needing no account, license, or per-repo quota. Activating that overlay is unaffected by anything below; this overlay only concerns *Pro*-tier features layered on top of it. -## What Pro adds, and why it isn't on - -Semgrep's **free/OSS CLI accepts a `languages: [powershell]` rule but silently -skips it** — 0 findings, no error, just a warning that it "requires Pro". Pro -also adds cross-file analysis, AI-assisted triage/remediation, and 60 AI -credits. Verified by testing against the running CLI (2026-07, semgrep 1.170.0). - -The blocker is not cost tuning, it's a **hard account-wide cap**: - -> Scan up to 10 repositories · Maximum 10 contributors - -This template is cloned into new repos on purpose. Baking a Pro dependency into -the template would mean every repo created from it silently competes for one of -only 10 slots on the account — a scarce resource, not a per-repo toggle. That is -why this stays an inert, deliberately-activated overlay: the decision to spend a -slot belongs to a human, once, per repo that actually needs PowerShell -enforcement (or the other Pro features) badly enough to justify it. +## What the hosted free tier adds, and why it isn't on + +Semgrep's current free hosted offering includes Pro rules, cross-file analysis, +and AI features, but its pricing page caps private repositories at 10 and its +usage policy caps private-project organizations at 10 monthly contributors. +Public projects have no contributor limit. These are service-plan constraints, +not limits on unauthenticated local Community Edition scans +([pricing](https://semgrep.dev/pricing/), +[usage and billing](https://semgrep.dev/docs/usage-limits/)). + +This template is cloned into new repositories on purpose, so it does not spend +hosted private-repository capacity by default. The current supported-language +table also does not list PowerShell; do not adopt this overlay merely on the +assumption that a Pro login adds PowerShell analysis +([supported languages](https://semgrep.dev/docs/supported-languages)). ## If you decide to activate it (on a specific repo) -1. Check the current slot count against the 10-repo cap before linking another - one — (or your org's usage page). +1. Check the current plan and private-repository/contributor usage before + linking another repository — (or your + organization's usage page). 2. Link the repo: , then `semgrep login` locally / in CI. -3. Rules with `languages: [powershell]` (or other Pro-only targets) then run - with `semgrep scan --pro` — the free-tier flag drops the Pro skip. -4. No PowerShell Pro rules exist in this repo yet — none are written or tested, - since there is currently no account to verify them against. Write and verify - them the same way the [bash rule](../../semgrep/rules/bash-no-blanket-strict-mode.yaml) +3. Run the rules for a language currently documented as supported, using the + product mode required by the chosen rules. +4. No PowerShell rules exist in this repo. Write and verify any future rule the + same way the [bash rule](../../semgrep/rules/bash-no-blanket-strict-mode.yaml) was: a violating + a clean sample file, run through `semgrep scan`, before adding the rule to `.config/semgrep/rules/`. ## Re-evaluate if -- The account already holds a Pro subscription for other reasons (then the - 10-repo cap is a real, trackable constraint to budget against, not a blocker). -- PowerShell policy enforcement becomes valuable enough on a *specific* repo to - spend one of the 10 slots on it deliberately. +- The account already has a suitable plan and available private-repository and + contributor capacity. +- Semgrep officially adds the target language and its analysis is verified + against representative positive and negative fixtures. diff --git a/.config/overlays/semgrep/README.md b/.config/overlays/semgrep/README.md index f7b5ee8..51a6f26 100644 --- a/.config/overlays/semgrep/README.md +++ b/.config/overlays/semgrep/README.md @@ -8,18 +8,20 @@ paste it in. ## Why activation is opt-in, not core It was briefly wired in as an active hook + scheduled CI workflow and then -reverted. Two real frictions drove that: - -- **The managed pre-commit hook does not work on Windows** — pre-commit clones - semgrep's own git repo, which contains case-colliding paths that fail - checkout on NTFS. The `repo: local` PyPI-wheel workaround below fixes it but - adds install weight and a `require_serial: true` workaround for a - `~/.semgrep/settings.yml` race between parallel instances. -- **Its language coverage has real gaps for this template's stack**: SQL has - no support at any tier; **PowerShell parses but Pro-gates rule execution** - (free CLI silently reports 0 findings — see - [`../semgrep-pro/README.md`](../semgrep-pro/README.md)), and Pro is capped at - 10 repos account-wide — unsuitable for a template cloned into many repos. +reverted. Two practical frictions drove that: + +- An earlier Windows test of Semgrep's managed pre-commit repository checkout + failed on case-colliding paths. This overlay therefore retains a + `repo: local` PyPI installation. Semgrep added native Windows support in + 2025 and now recommends `pipx` on Windows, so this historical managed-hook + failure should be revalidated before treating it as a current platform + limitation ([Semgrep installation guidance](https://semgrep.dev/docs/update), + [native Windows announcement](https://semgrep.dev/blog/2025/semgrep-community-edition-fall-release-2025)). +- This template is PowerShell-heavy, while Semgrep's current supported-language + table does not list PowerShell or SQL. The shipped local rules cover only + languages that were actually verified here; PSScriptAnalyzer remains the + PowerShell policy engine + ([Semgrep supported languages](https://semgrep.dev/docs/supported-languages)). None of that makes semgrep *bad* — the shipped rules are verified and work — but it does not clear the bar for base-template tooling every clone inherits by @@ -32,8 +34,8 @@ default. Tracking issue for a replacement evaluation: 2. **Wire the hook** — paste the block from [`precommit-hook.yaml`](precommit-hook.yaml) into the `repos:` list of [`/.pre-commit-config.yaml`](../../../.pre-commit-config.yaml). It installs - from PyPI as a `repo: local` hook (not the managed `repo:` clone, which - fails on Windows — see above). + from PyPI as a `repo: local` hook, preserving the configuration that was + verified on this repository. 3. **Verify** — `semgrep scan --config .config/semgrep/rules --metrics=off .` should run clean, or report only intended findings. diff --git a/.config/scripts/README.md b/.config/scripts/README.md index 3df59d1..072e2ca 100644 --- a/.config/scripts/README.md +++ b/.config/scripts/README.md @@ -58,10 +58,12 @@ cost/risk — *not* by a unit/integration label. there are none. Standard/thorough hooks ship **commented-in-place** in the pre-commit config — uncomment when you have tests for them. + Tests are **co-located** beside the script they cover (`Foo.Tests.ps1` next to `Foo.ps1`) — the repo-wide default. The exception is [`setup/`](../../setup/), a user-facing runbook that segregates its tests into `setup/tests/`; discovery recurses from the repo root, so either layout runs the same. + Keep the fast (commit) lane genuinely fast, or contributors reach for `--no-verify`. The floor is pwsh + Pester warm-up (~1-3s), not your assertions. diff --git a/.config/semgrep/README.md b/.config/semgrep/README.md index 4796b85..9feb815 100644 --- a/.config/semgrep/README.md +++ b/.config/semgrep/README.md @@ -7,14 +7,15 @@ semgrep into pre-commit is opt-in: see [`.config/overlays/semgrep/README.md`](../overlays/semgrep/README.md) to activate. -Scope boundaries: PowerShell parses but its rules are **Pro-gated** (free-tier -CLI silently skips them — see -[`../overlays/semgrep-pro/README.md`](../overlays/semgrep-pro/README.md) -before assuming a `languages: [powershell]` rule here does anything). SQL has -no semgrep support at any tier. pwsh policies live in +Scope boundaries: Semgrep's current +[supported-language table](https://semgrep.dev/docs/supported-languages) does +not list PowerShell or SQL. PowerShell policies therefore live in [`../PSScriptAnalyzerRules/`](../PSScriptAnalyzerRules/) instead — that -mechanism is active in the base template today. Secrets are gitleaks; workflow -security is zizmor. +mechanism is active in the base template today. Do not infer support merely +because a parser accepts a language name; verify a positive and negative +fixture against the current engine. Secrets are gitleaks; workflow security is +zizmor. The hosted-service trade-offs are recorded in +[`../overlays/semgrep-pro/README.md`](../overlays/semgrep-pro/README.md). ## Rules shipped here diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index db6356b..e5b097f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,6 +4,6 @@ /.github/ @DenWin /.pre-commit-config.yaml @DenWin /.config/ @DenWin -/setup/ @DenWin +/setup/ @DenWin # setup-teardown:template-only /AGENTS.md @DenWin /CLAUDE.md @DenWin diff --git a/.github/README.md b/.github/README.md index 767cb09..0641d84 100644 --- a/.github/README.md +++ b/.github/README.md @@ -18,9 +18,12 @@ File-scoped linters run only over changed files on a PR; the test hooks are `always_run`, so the fast lane runs in full regardless. Third-party actions are **pinned by full commit SHA** (with a `# vN` comment for -readability) — a mutable tag can be re-pointed at malicious code, a SHA cannot. -Dependabot updates the pins and refreshes the comment. The `zizmor` pre-commit -hook enforces this and other workflow-security rules. +readability), following GitHub's recommendation to pin actions to a full-length +commit SHA as the only immutable release reference +([GitHub secure-use guidance](https://docs.github.com/en/enterprise-cloud@latest/actions/reference/security/secure-use)). +Dependabot updates those SHA pins and version comments +([supported ecosystems](https://docs.github.com/en/code-security/reference/supply-chain-security/supported-ecosystems-and-repositories)). +The `zizmor` pre-commit hook enforces this and other workflow-security rules. The `lint` job's first step is a **full-history secret scan** (`gitleaks-action`). This is the one CI check that is *not* a local pre-commit hook, by design: the @@ -30,8 +33,10 @@ the whole history / PR range. It runs the same gitleaks engine, so no lint logic is duplicated. `GITLEAKS_VERSION` in the workflow is kept in sync with the managed hook revision so rule definitions do not drift between local and CI scans. Org-owned repos need a free `GITLEAKS_LICENSE` secret; user-owned repos -do not. +do not, per the +[Gitleaks Action licensing notes](https://github.com/gitleaks/gitleaks-action#license). + ### Protecting the default branch (once per repo) The `merge_group` gate only fires if the branch has a **merge-queue ruleset**. @@ -44,6 +49,9 @@ pwsh -NoProfile -File setup/Protect-MainBranch.ps1 Run it **after** the first push and one CI run (so the `lint` check exists), then delete the `setup/` folder. See [`setup/README.md`](../setup/README.md). +The trigger and ruleset relationship is documented by +[GitHub's merge-queue guide](https://docs.github.com/en/enterprise-cloud@latest/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue). + ### What CI installs @@ -57,10 +65,13 @@ self-install: ## `CODEOWNERS` Auto-requests review from the listed owners when matching paths change; the -sensitive control surfaces (`.github/`, `.pre-commit-config.yaml`, `.config/`, -`AGENTS.md`, `setup/`) are called out explicitly. It only *requests* review -until "require code owner review" is turned on in the branch ruleset — left off -for solo repos, since you cannot approve your own PR. +sensitive permanent control surfaces (`.github/`, +`.pre-commit-config.yaml`, `.config/`, `AGENTS.md`, `CLAUDE.md`) are called out +explicitly. CODEOWNERS requests review by itself; enforcement requires a branch +rule that requires code-owner review +([GitHub documentation](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners)). +That enforcement is left off for solo repos, since you cannot approve your own +PR. ## `pull_request_template.md` @@ -75,4 +86,6 @@ is deliberately narrow — the only ecosystem in this template it can maintain i **GitHub Actions versions** (`github-actions`, monthly). pre-commit hook pins are updated by `pre-commit autoupdate`; PowerShell/npm deps are handled elsewhere. A 7-day `cooldown` delays adopting a just-published version, leaving a window for -a compromised or yanked release to be caught upstream. +a compromised or yanked release to be caught upstream. The option and its +behavior are defined in the +[Dependabot options reference](https://docs.github.com/en/code-security/reference/supply-chain-security/dependabot-options-reference#cooldown-). diff --git a/AGENTS.md b/AGENTS.md index d87dba6..04d1bd8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,19 +16,21 @@ PRs, full at the merge gate. ## Mechanism map -| Mechanism | Where | Doc | -| ------------------- | -------------------------------- | --------------------------------------------------------------------- | -| Orchestration | `.pre-commit-config.yaml` (root) | this file + inline comments | -| Configs | `.config/` | [`.config/README.md`](.config/README.md) | -| Linting & testing | `.config/scripts/` | [`.config/scripts/README.md`](.config/scripts/README.md) | -| CI & automation | `.github/` | [`.github/README.md`](.github/README.md) | -| Editor integration | `.vscode/` | [`.vscode/README.md`](.vscode/README.md) | -| Policy rules | `.config/PSScriptAnalyzerRules/` | [`.config/scripts/README.md`](.config/scripts/README.md) | -| Opt-in tooling | `.config/overlays/` | [`.config/overlays/vale/README.md`](.config/overlays/vale/README.md) | -| Documentation graph | Markdown docs + README indexes | [`docs/knowledge-format.md`](docs/knowledge-format.md) | -| AI delegation | `.claude/`, `.codex/` | [`CLAUDE.md`](CLAUDE.md) + [`.codex/config.toml`](.codex/config.toml) | +| Mechanism | Where | Doc | +| ------------------- | ---------------------------------- | --------------------------------------------------------------------- | +| Orchestration | `.pre-commit-config.yaml` (root) | this file + inline comments | +| Configs | `.config/` | [`.config/README.md`](.config/README.md) | +| Linting & testing | `.config/scripts/` | [`.config/scripts/README.md`](.config/scripts/README.md) | +| CI & automation | `.github/` | [`.github/README.md`](.github/README.md) | +| Editor integration | `.vscode/` | [`.vscode/README.md`](.vscode/README.md) | +| Policy rules | `.config/PSScriptAnalyzerRules/` | [`.config/scripts/README.md`](.config/scripts/README.md) | +| Opt-in tooling | `.config/overlays/` | [`.config/overlays/README.md`](.config/overlays/README.md) | +| Documentation graph | Markdown/AsciiDoc + README indexes | [`docs/README.adoc`](docs/README.adoc) | +| AI delegation | `.claude/`, `.codex/` | [`CLAUDE.md`](CLAUDE.md) + [`.codex/config.toml`](.codex/config.toml) | + | One-time repo setup | `setup/` (delete after use) | [`setup/README.md`](setup/README.md) | -| Optional AI skills | `setup/optional-skills/` | [`setup/README.md`](setup/README.md) | +| Optional AI skills | `setup/optional-skills/` | [`setup/optional-skills/README.adoc`](setup/optional-skills/README.adoc) | + Root convention files are declarative and self-documenting: `.editorconfig` (style), `.gitattributes` (eol=lf), `.gitignore`, `.claudeignore`. @@ -51,6 +53,7 @@ root). Key principles encoded there: Behavioral changes to the repo's scripts are developed test-first with the `develop-with-tdd` skill. If that skill — or an equivalent TDD skill — is already installed in your agent, use it; there is no need for a second copy. + If none exists, use the bundled one at [`setup/optional-skills/skills/develop-with-tdd/`](setup/optional-skills/skills/develop-with-tdd/SKILL.md): install it into your agent's skill directory (Claude Code: `.claude/skills/` @@ -58,17 +61,19 @@ in the repo, or `~/.claude/skills/` for all repos), or follow its `SKILL.md` in place. The bundle is deleted with the rest of `setup/` at the end of bootstrap — install it *before* running `Complete-Setup.ps1` if you want to keep it. + ## Documentation relationships This repo uses a deliberately reduced, non-conformant subset of the draft Open -Knowledge Format: ordinary Markdown links make related documents discoverable, -and README indexes provide progressive disclosure without turning each -consumer repo into a knowledge bundle. The adopted and rejected conventions -are recorded in +Knowledge Format: ordinary Markdown links and AsciiDoc cross-references make +related documents discoverable, and README indexes provide progressive +disclosure without turning each consumer repo into a knowledge bundle. The +adopted and rejected conventions are recorded in [`docs/knowledge-format.md`](docs/knowledge-format.md); do not copy the upstream draft into this repo. + **Keep the migration runbook connected.** When a change adds, removes, or materially changes a permanent mechanism, agent entry point, delegation file, documentation convention, or setup step, review @@ -76,6 +81,7 @@ documentation convention, or setup step, review the retrofit steps change; otherwise state in the PR why migration is unaffected. This rule lives here because Codex reads `AGENTS.md` directly and Claude imports it through `CLAUDE.md`. + ## Reuse: prefer an include over a copy; a symlink only when no include exists @@ -87,16 +93,17 @@ preference: config paths in pre-commit hook `args`. Degrades *loudly* (a missing file errors) and works on every machine — use it whenever the format offers one. 2. **Symlink** — when the format has no include mechanism. One source of - truth, no drift (ai-lab copied such files and they drifted). Git tracks - symlinks natively, but on Windows they only materialize with **Developer - Mode** (Settings → For developers) *and* `git config core.symlinks true` — - without both, git silently checks them out as plain text files holding the - target path, and nothing errors. `Initialize-DevEnvironment.ps1` probes - symlink creation and warns when the machine can't do it. Do **not** reach - for an elevated shell as a workaround (that breaks pre-commit, see the - bootstrap note). + truth, no copy drift. Git documents that `core.symlinks=false` checks + symlinks out as small plain files containing the link text + ([Git documentation](https://git-scm.com/docs/git-config#Documentation/git-config.txt-coresymlinks)). + On Windows, creating symlinks without elevation also depends on Developer + Mode ([Microsoft documentation](https://learn.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development)). + `Initialize-DevEnvironment.ps1` probes creation and warns when the machine + cannot do it. Do **not** reach for an elevated shell as a workaround (that + breaks pre-commit, see the bootstrap note). 3. **Copy** — only when the two files are meant to diverge. + ## Retrofit an existing repo See [`setup/MIGRATION.md`](setup/MIGRATION.md) for the full step-by-step walkthrough of @@ -104,6 +111,7 @@ bringing this architecture into a repo not scaffolded from this template — one step per concept in the mechanism map above, in dependency order, including the platform limitations you'll hit along the way (e.g. `merge_queue` on a personal/user-owned repo). + ## Opening a pull request diff --git a/CLAUDE.md b/CLAUDE.md index 763baaf..4173211 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,13 +10,15 @@ defines the TDD workflow and where the bundled skill lives. Reserve the lead model (Fable/Opus) for reasoning-heavy work: architecture, debugging, code review, and anything touching `.pre-commit-config.yaml` -semantics or policy rules. Delegate the following **named task types** to -cheaper models via the Task tool instead of doing them inline: +semantics or policy rules. Delegate the following **named task types** when +isolation or parallel work justifies the extra tokens; the trade-off is recorded +in [the token-saving knowledge base](docs/token-saving.adoc): -- **Git commits** → `committer` subagent (haiku, defined in - `.claude/agents/committer.md`). The lead model writes the commit message - and passes it plus the explicit file list in the delegation prompt; the - subagent only stages and commits. +- **Git commits** are eligible for the `committer` subagent (haiku, defined in + `.claude/agents/committer.md`) when it preserves scarce lead context, + overlaps useful lead work, or the user requests isolation. For a tiny + sequential commit, the lead may commit directly. The lead writes the commit + message and explicit file list; the subagent only stages and commits. - **PR descriptions / `gh pr create`** → general-purpose subagent, `model: sonnet`. Must fill every section of `.github/pull_request_template.md`, including Evidence. diff --git a/README.md b/README.md index 0f4a26f..b8b6d6f 100644 --- a/README.md +++ b/README.md @@ -15,5 +15,8 @@ pwsh -NoProfile -File .config/scripts/Initialize-DevEnvironment.ps1 ``` For what the tooling does and why it's shaped this way, see -[.config/README.md](.config/README.md). Migrating this tooling into a repo -that isn't scaffolded from this template? See [setup/MIGRATION.md](setup/MIGRATION.md). +[.config/README.md](.config/README.md). + +Migrating this tooling into a repo that isn't scaffolded from this template? +See [setup/MIGRATION.md](setup/MIGRATION.md). + diff --git a/docs/README.adoc b/docs/README.adoc new file mode 100644 index 0000000..ae44f87 --- /dev/null +++ b/docs/README.adoc @@ -0,0 +1,12 @@ += Documentation index + +The permanent knowledge documents in this repository are: + +* link:knowledge-format.md[Knowledge-format convention] — the deliberately + reduced, OKF-inspired rules used by Markdown and AsciiDoc documents. +* xref:token-saving.adoc[Token-saving options for coding agents] — ways to + reduce total usage or preserve context, including when subagent delegation + is counterproductive. + +The repository architecture and its per-mechanism documentation map start in +link:../AGENTS.md[AGENTS.md]. diff --git a/docs/knowledge-format.md b/docs/knowledge-format.md index caca108..e3262e8 100644 --- a/docs/knowledge-format.md +++ b/docs/knowledge-format.md @@ -17,22 +17,26 @@ than copying its rules into this repository. Borrow these useful properties: -- Write human- and agent-readable Markdown that remains easy to review as a Git - diff, reflecting OKF's +- Write human- and agent-readable Markdown or AsciiDoc that remains easy to + review as a Git diff, reflecting OKF's [readability and diffability goals](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md#1-motivation). -- Express relationships with explicit, standard Markdown links and enough +- Express relationships with explicit standard links (`[label](path)` in + Markdown; `xref:path[label]` or `link:path[label]` in AsciiDoc) and enough surrounding prose to explain the relationship, as in [OKF cross-linking](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md#5-cross-linking). -- Use a `README.md` as the index for a documentation directory when an index - helps readers discover its contents before opening individual files. This is - the local equivalent of OKF's +- Use `README.md` or `README.adoc` as the index for a documentation directory + when an index helps readers discover its contents before opening individual + files. This is the local equivalent of OKF's [progressive-disclosure index](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md#6-index-files). -- Cite external sources for claims that depend on them, using ordinary Markdown - links near the claim or a compact citations section. OKF likewise treats +- Cite external sources for claims that depend on them, using ordinary links + near the claim or a compact citations section. OKF likewise treats [citations as supporting links](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md#8-citations). -Use ordinary repository-relative links such as `../README.md` or -`./design.md`. Do not use OKF's leading-slash, bundle-relative link convention. +AsciiDoc is not an exception to the convention: it follows the same +discoverability, progressive-disclosure, and citation rules as Markdown, using +native AsciiDoc syntax. Use ordinary repository-relative targets such as +`../README.md`, `xref:../guide/README.adoc[]`, or `link:../SKILL.md[]`. Do not +use OKF's leading-slash, bundle-relative link convention. ## Deliberate omissions diff --git a/docs/token-saving.adoc b/docs/token-saving.adoc new file mode 100644 index 0000000..91b9b6d --- /dev/null +++ b/docs/token-saving.adoc @@ -0,0 +1,130 @@ += Token-saving options for coding agents + +== What is being saved? + +Do not treat these as interchangeable. Reducing a main conversation's context +does *not* necessarily reduce total tokens or cost. + +[cols=",,",options="header",] +|=== +|Objective |Practical levers |Important trade-off +|Total tokens and API cost |Avoid needless turns, file/log reads, and always-on tools; use a suitable lower-cost model for simple work |Less investigation or reasoning can cause expensive rework. +|Context-window headroom |Start a fresh task, compact a continuing task, use path-scoped instructions/skills, and isolate noisy work in a subagent |Compaction is lossy; a subagent still consumes its own tokens. +|Latency |Limit unnecessary tools/context and run independent work concurrently |Parallel agents increase total work and may contend for limits. +|Quality |Give a precise goal, constraints, acceptance tests, and a small project map |A short but ambiguous prompt often costs _more_ through exploration and retries. +|=== + +== High-value practices + +[arabic] +. *Make the first request decision-complete.* State the desired outcome, +scope, non-goals, relevant files, and verification command. GitHub observes +that vague tasks force extra inference and exploration, while useful +instructions can reduce the number of runs needed. This is usually the +strongest total-token and quality lever. https://docs.github.com/en/copilot/tutorials/optimize-ai-usage[GitHub guidance] +. *Keep durable guidance small and routed.* For Codex, keep `AGENTS.md` +concise and add directory-specific routing only where recurring mistakes or +unnecessary reading justify it. Codex explicitly recommends keeping it small +and says routing guidance can address excessive document reading. +https://developers.openai.com/codex/concepts/customization[OpenAI: Customization] +Copilot CLI similarly cautions that large general instructions can distract +from the task; use a skill or agent for conditional detail instead. +https://docs.github.com/en/copilot/concepts/agents/copilot-cli/comparing-cli-features[GitHub: feature comparison] +. *Use progressive disclosure.* Put specialized procedures, examples, and +helper scripts in skills, not every-session instructions. Codex loads skill +details only when needed; Copilot recommends skills for detailed, +task-specific guidance and custom instructions for short, universal rules. +https://learn.chatgpt.com/docs/glossary[OpenAI glossary] +https://docs.github.com/en/copilot/how-tos/copilot-cli/customize-copilot/add-skills[GitHub: agent skills] +This preserves main-window headroom and can reduce irrelevant input tokens; +it is not a saving if a broad skill is invoked on every task. +. *Manage a long thread deliberately.* For unrelated work, start a new +conversation. For continuing work, compact with a task-specific focus and +retain a short handoff/plan with decisions and validation state. Copilot CLI +provides `/context` and `/compact`; Claude Code's `/compact` replaces the +conversation with a structured summary. https://docs.github.com/en/copilot/tutorials/optimize-ai-usage[GitHub: optimizing AI usage] +https://code.claude.com/docs/en/context-window[Anthropic: context window] +This preserves context capacity, but a summary can omit detail, so keep +source-of-truth decisions in the repository rather than relying on it. +. *Reduce always-present tool surface.* Enable only the MCP tools/toolsets +needed for the task: tool definitions consume context. GitHub calls this out +explicitly, and Claude Code documents that MCP names and skill descriptions +load before work begins. https://docs.github.com/en/copilot/tutorials/optimize-ai-usage[GitHub: optimizing AI usage] +https://code.claude.com/docs/en/context-window[Anthropic: context window] +This reduces input/context and may improve latency; it can reduce quality if +it removes a necessary source of evidence. +. *Cap noisy inputs at the source.* Ask tools for a focused path, diff, test +failure, or log excerpt rather than a repository-wide dump. Where exposed, +use summarizing tools: GitHub's example skill summarizes failed-job logs +rather than filling the context with thousands of lines. +https://docs.github.com/en/copilot/how-tos/copilot-cli/customize-copilot/add-skills[GitHub: agent skills] + +== Product-specific notes + +* *Codex:* combine a compact `AGENTS.md` with on-demand skills. Codex defines +context as files, prior messages, tool output, and instructions, and supports +compaction across its coding surfaces. https://learn.chatgpt.com/docs/glossary[OpenAI glossary] +Its customization docs say skills keep rich workflows available without +upfront context bloat. https://developers.openai.com/codex/concepts/customization[OpenAI: Customization] +For a well-scoped, simple task, lower reasoning can reduce usage at a quality +risk; conversely, Codex Fast mode is a latency option, not a saving—it uses +more credits for faster responses. https://learn.chatgpt.com/docs/agent-configuration/speed[OpenAI: Codex speed] +* *Claude Code:* use `/compact` for a continuing task, `/clear` or a new +session for a new task, and keep `CLAUDE.md`, auto-memory, MCP, and skill +setup lean. The context documentation shows these items loading before work +and explains what compaction retains. https://code.claude.com/docs/en/context-window[Anthropic: context window] +* *Copilot CLI:* check `/context`, use `/compact` for a long related session, +and `/new` or `/clear` when switching work. Use minimal custom instructions, +path-scoped instruction files, and just-in-time skills. https://docs.github.com/en/enterprise-cloud@latest/copilot/concepts/agents/copilot-cli/context-management[GitHub: context management] +https://docs.github.com/en/copilot/concepts/agents/copilot-cli/comparing-cli-features[GitHub: feature comparison] +* *GitHub Copilot generally:* improve the repo map and prompt rather than +attaching broad files repeatedly; context sent to Copilot counts as input +tokens. https://docs.github.com/en/copilot/tutorials/optimize-ai-usage[GitHub: optimizing AI usage] + +== Delegating mechanical commits to a lower-cost subagent + +*Conclusion: this chiefly preserves the parent context; it does not normally +reduce total tokens.* Codex, Claude Code, and Copilot CLI all document that a +subagent has a separate context window, and Claude Code/Copilot return a +summary to the parent. https://learn.chatgpt.com/docs/agent-configuration/subagents[OpenAI: Codex subagents] +https://code.claude.com/docs/en/sub-agents[Anthropic: subagents] +https://docs.github.com/en/copilot/concepts/agents/copilot-cli/about-custom-agents[GitHub: custom agents] + +That isolation keeps `git status`, diffs, staging, hooks, and commit output out +of the main window, but it adds a new task prompt, agent setup, tool calls, and +a report. Thus the total-token claim is an inference from the documented +separate-agent architecture: delegation generally *shifts and adds* context +rather than eliminating it. A cheaper subagent can lower _monetary_ cost only +when its per-token price/credit rate is lower; it does not make the work free. +It can lower elapsed time only when it genuinely runs in parallel with useful +parent work. + +*Recommendation:* keep mechanical commit delegation as an opt-in, narrow +pattern—not a default. Use it when the parent is doing substantial independent +work, its context is scarce, or a cheap agent has an explicit file list, +pre-approved commit message, and no scope decisions. Otherwise, have the main +agent commit directly. In either case, require the final actor to verify the +diff, staged paths, hook result, commit SHA, and clean/expected worktree; a +weak low-cost agent can erase savings through a wrong commit or a repair turn. + +== Pinning a larger model or higher reasoning + +Codex custom agents may set both `model` and `model_reasoning_effort`; omitted +values inherit from the parent +(https://learn.chatgpt.com/docs/agent-configuration/subagents#custom-agents[OpenAI: +custom agents]). That makes an explicit high-reasoning reviewer or architect a +valid design. It is wise when the task's error cost and synthesis difficulty +justify it, not as a blanket setting: higher reasoning spends more budget and a +pinned model name adds maintenance when catalogs change. + +Prefer inheritance for mechanical agents such as a committer. Pin a model only +when the role genuinely needs a stable capability/cost profile, record why, and +periodically verify the name. Current Codex documentation describes inheritance +and overrides but does not promise that an unavailable custom-agent model will +silently fall back; treat model availability as a validation and maintenance +responsibility. + +For API-backed Claude workflows specifically, model choice, prompt caching, and +batching are documented monetary levers; batching trades latency for a 50% +input/output discount, and cache reads are priced below base input tokens. +https://docs.anthropic.com/en/docs/about-claude/pricing[Anthropic pricing] diff --git a/setup/AI-Maintainer-Identity.adoc b/setup/AI-Maintainer-Identity.adoc index 01ef5f6..c6b7ab5 100644 --- a/setup/AI-Maintainer-Identity.adoc +++ b/setup/AI-Maintainer-Identity.adoc @@ -41,10 +41,13 @@ the ones this document is for: `GH_TOKEN`/git credential of the shell they run in. Create *one App (or PAT) per agent* — separate identities keep attribution clean and let you revoke one agent without touching the others. -* *Vendor-hosted agents* (GitHub Copilot coding agent, Codex cloud) act - through the vendor's own pre-built GitHub App. You install and repo-scope - *theirs* — you cannot substitute your own. Verify their installation is - limited to the intended repos and grants no admin. +* *Vendor-hosted agents* use their product's supported source-control + integration rather than the local shell credential described here. Configure + and repository-scope that integration according to the vendor documentation. + Codex cloud, for example, uses connected source repositories and respects the + repository permissions and protections exposed by the connected source + system + (https://learn.chatgpt.com/docs/enterprise/admin-setup#step-5-configure-codex-cloud[OpenAI documentation]). Verification is scriptable either way: `Test-AIMaintainerIdentity.ps1`, run from the agent's shell, fails unless the active credential is an installation @@ -59,7 +62,8 @@ token or fine-grained PAT with write/push access and without admin. | What it is | An installable app whose installation token acts as a bot identity (`your-app[bot]`). -| A scoped personal access token owned by you or a dedicated bot account. +| A repository-scoped token owned by a separate bot account that is a member of + the repository-owning organization. | Attribution | Commits/PRs show the app's bot identity — clearly non-human. @@ -79,21 +83,26 @@ token or fine-grained PAT with write/push access and without admin. | Best when | You want durable, auditable, revocable automation identities. -| You want the quickest path and accept token management. +| The repository is organization-owned and the bot is an organization member. |=== -Recommendation: *GitHub App* for durable agents; a *fine-grained PAT on a bot -account* if you want to move fast today. Never use a classic (non-fine-grained) -PAT, and never grant `Administration`. +Recommendation: use a *GitHub App* for personal-account repositories and for +durable automation generally. A fine-grained PAT is an option only for an +organization-owned repository when the dedicated bot account is an +organization member. GitHub documents that fine-grained PATs cannot access +repositories where the user is only an outside or repository collaborator +(https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#fine-grained-personal-access-tokens-limitations[GitHub limitations]). +Never use a classic PAT, and never grant `Administration`. == Option A — GitHub App === Create -`New-AIMaintainerApp.ps1` automates this section via GitHub's app-manifest -flow — it builds the manifest below, opens the browser for the single -confirmation GitHub requires (there is no pure-API path to App creation), and -saves the private key locally. The manual path, for reference: +`New-AIMaintainerApp.ps1` automates this section via GitHub's documented +https://docs.github.com/en/apps/sharing-github-apps/registering-a-github-app-from-a-manifest[App-manifest flow]: +it builds the manifest below, opens the browser for confirmation, exchanges the +one-time code, and saves the private key locally. The manual path, for +reference: . *Settings → Developer settings → GitHub Apps → New GitHub App.* . Name it per agent or one shared maintainer app (e.g. `denwin-ai-maintainer`). @@ -120,42 +129,55 @@ keep that behind `CODEOWNERS` review instead. === Use from an agent -An App authenticates as itself (JWT), then exchanges that for a short-lived -*installation access token*. The simplest path is the `gh` token helper or the -`actions/create-github-app-token` action in CI. Locally: +An App authenticates as itself with a JWT, then exchanges that for an +installation access token. GitHub documents a one-hour installation-token +expiry and supports restricting the token to selected repositories and +permissions +(https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/generating-an-installation-access-token-for-a-github-app[GitHub documentation]). +This setup bundle provides the local token helper: -[source,bash] +[source,powershell] ---- -# Mint an installation token (expires in ~1 hour), then point git/gh at it. -# Use a helper such as `gh-token` or a small script signing a JWT with the -# app private key; store the result in GH_TOKEN for the agent's shell only. -export GH_TOKEN="" -git -c user.name="denwin-ai-maintainer[bot]" \ - -c user.email="+denwin-ai-maintainer[bot]@users.noreply.github.com" \ - commit -m "…" +$env:GH_TOKEN = pwsh -NoProfile -File setup/New-AIMaintainerToken.ps1 ` + -AppId ` + -InstallationId ` + -Repository owner/name ` + -PrivateKeyPath ~/.github-apps/my-agent.private-key.pem + +gh auth setup-git +pwsh -NoProfile -File setup/Test-AIMaintainerIdentity.ps1 ---- -Give the agent a shell where `GH_TOKEN` is the installation token — separate from -your personal `gh auth login` session. +Give the agent only this shell. The token is scoped to the named repository and +is separate from your personal `gh auth login` session. Mint a new one after +expiry; never persist it in the repository or shell profile. == Option B — Fine-grained PAT -. Create (or reuse) a dedicated *bot GitHub account*; add it as a repository - *collaborator with Write* (not Admin). +. This option requires an *organization-owned repository*. Create (or reuse) a + dedicated bot GitHub account and add it as an organization member with only + the repository access it needs. . As that account: *Settings → Developer settings → Fine-grained tokens → Generate new token.* -. *Resource owner* = you; *Repository access* = only this repo. +. *Resource owner* = the organization; *Repository access* = only this repo. . *Permissions* — Contents: Read and write; Pull requests: Read and write; Metadata: Read-only. Nothing else. No Administration. . Set a short expiry and calendar a rotation. . Configure the agent's git/`gh` to use that token and the bot account's `user.name`/`user.email`. +Do not use this path for a repository owned by a personal account: GitHub's +fine-grained PAT collaborator limitation prevents the separate bot account from +using such a token there. Use the GitHub App path instead. + == Harden further (optional) -* *Require signed commits* — add the `required_signatures` rule to the branch - ruleset so only verified identities' commits are accepted. GitHub Apps sign - automatically; a bot account needs a GPG/SSH signing key. +* *Require signed commits* only after proving the actual commit path. A local + `git commit` authenticated with an App token is not automatically signed; + configure GPG, SSH, or S/MIME signing for local commits. GitHub's special bot + verification rules apply to commits created through authenticated API + requests under specific author/committer conditions + (https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification#signature-verification-for-bots[GitHub documentation]). * *Restrict who can push to `main`* — with a PR-only ruleset and no bypass actors, no identity (including yours) pushes directly; break-glass is a deliberate, logged admin action. @@ -173,8 +195,15 @@ Mitigate by convention: == See also -* `setup/New-AIMaintainerApp.ps1` — scripted App creation (manifest flow). -* `setup/Test-AIMaintainerIdentity.ps1` — verify an agent shell's credential; run before `Complete-Setup.ps1`. -* `setup/Protect-MainBranch.ps1` — the branch ruleset the identities are enforced against. -* `setup/Enable-RepoSecurity.ps1` — server-side security toggles. -* `.github/CODEOWNERS` — the control surfaces that require owner review. +* xref:New-AIMaintainerApp.ps1[New-AIMaintainerApp.ps1] — scripted App creation + through the manifest flow. +* xref:New-AIMaintainerToken.ps1[New-AIMaintainerToken.ps1] — mint one + repository-scoped installation token. +* xref:Test-AIMaintainerIdentity.ps1[Test-AIMaintainerIdentity.ps1] — verify an + agent shell's credential; run before `Complete-Setup.ps1`. +* xref:Protect-MainBranch.ps1[Protect-MainBranch.ps1] — the branch ruleset the + identities are enforced against. +* xref:Enable-RepoSecurity.ps1[Enable-RepoSecurity.ps1] — server-side security + toggles. +* link:../.github/CODEOWNERS[CODEOWNERS] — the control surfaces that require + owner review. diff --git a/setup/CHANGELOG.md b/setup/CHANGELOG.md index 10b632c..18c9bfe 100644 --- a/setup/CHANGELOG.md +++ b/setup/CHANGELOG.md @@ -8,6 +8,38 @@ until one is cut. Rationale lives with each mechanism's doc ([`/AGENTS.md`](../AGENTS.md), per-folder READMEs, script headers) — this file records only what changed. +## [0.4.0] + +### Added + +- `New-AIMaintainerToken.ps1` for cryptographically signed, + repository-scoped GitHub App installation tokens. +- Teardown markers and permanent-document cleanup in `Complete-Setup.ps1`. +- Indexes for optional skills, overlays, and permanent docs. +- A sourced token-saving knowledge base and three retained TDD assessment + artifacts. + +### Changed + +- The complete human TDD knowledge base is now AsciiDoc; runtime skill files + remain Markdown. +- Markdown and AsciiDoc follow the same reduced OKF-inspired documentation + convention. +- Mechanical committer delegation is opt-in when isolation or parallel work + justifies its additional tokens. +- AI-maintainer guidance now recommends GitHub Apps for personal repositories, + documents fine-grained PAT collaborator limits, and distinguishes local + commit signing from API-created bot verification. +- The migration runbook covers the updated documentation, overlay, optional + skill, and AI-maintainer mechanisms. + +### Fixed + +- `Complete-Setup.ps1` now verifies the AI-maintainer identity and removes + dangling `setup/` references from permanent documents. +- Semgrep and GitHub behavior claims now link to current primary sources and no + longer present historical Windows or plan constraints as universal. + ## [0.3.0] ### Added diff --git a/setup/Complete-Setup.ps1 b/setup/Complete-Setup.ps1 index 23c958c..6c0d801 100644 --- a/setup/Complete-Setup.ps1 +++ b/setup/Complete-Setup.ps1 @@ -12,9 +12,10 @@ 1. FAILSAFE: aborts if the repo is a template repo (`isTemplate`), so the template itself never loses its setup tooling, - 2. aborts on a dirty worktree, so the PR contains only the removal, - 3. branches, `git rm -r setup`, commits, pushes, and opens a PR whose - body fills the repo's PR template (Evidence included). + 2. aborts on a dirty worktree or unsafe AI-maintainer identity, + 3. branches, removes template-only references from permanent docs, + runs `git rm -r setup`, commits, pushes, and opens a PR whose body + fills the repo's PR template (Evidence included). Requires the GitHub CLI, authenticated with push access, run from the repo root. @@ -31,6 +32,63 @@ Set-StrictMode -Version Latest function Test-GhCli { [bool](Get-Command gh -ErrorAction SilentlyContinue) } +function Test-AIMaintainerPrecondition { + param([string]$RepositoryRoot = (Get-Location).Path) + + $identityScript = Join-Path $RepositoryRoot 'setup' 'Test-AIMaintainerIdentity.ps1' + if (-not (Test-Path -LiteralPath $identityScript -PathType Leaf)) { + Write-Error "Identity verifier not found: $identityScript" + return $false + } + + & pwsh -NoProfile -File $identityScript + return $LASTEXITCODE -eq 0 +} + +function Remove-TemplateOnlyDocumentation { + <# Remove content explicitly marked as useful only while setup/ exists. + Markers let downstream repos customize the surrounding documents + without this script replacing whole files. Returns changed paths. #> + [CmdletBinding(SupportsShouldProcess)] + param( + [string]$RepositoryRoot = (Get-Location).Path, + [string[]]$RelativePaths = @( + 'AGENTS.md', + 'README.md', + '.github/README.md', + '.github/CODEOWNERS', + '.config/scripts/README.md' + ) + ) + + $changed = [System.Collections.Generic.List[string]]::new() + $blockPattern = '(?ms)^[ \t]*\r?\n.*?^[ \t]*\r?\n?' + $linePattern = '(?m)^.*(?:|# setup-teardown:template-only).*(?:\r?\n|$)' + + foreach ($relativePath in $RelativePaths) { + $path = Join-Path $RepositoryRoot $relativePath + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { + continue + } + + $original = [System.IO.File]::ReadAllText($path) + $updated = [regex]::Replace($original, $blockPattern, '') + $updated = [regex]::Replace($updated, $linePattern, '') + + if ($updated -ne $original -and + $PSCmdlet.ShouldProcess($path, 'Remove template-only documentation')) { + [System.IO.File]::WriteAllText( + $path, + $updated, + [System.Text.UTF8Encoding]::new($false) + ) + $changed.Add($relativePath) + } + } + + $changed +} + function Test-RemovalAllowed { <# The template failsafe (pure; no side effects): a template repo keeps its setup/ folder — only repos CREATED from it remove theirs. #> @@ -56,13 +114,15 @@ in place. ## Scope -Deletes `setup/` only. No other file changes. +Deletes `setup/` and removes its template-only references from permanent +documentation and `CODEOWNERS`. ## Risk & rollback -Low — the folder is inert documentation/tooling once setup ran; nothing -references it at runtime. Rollback: revert this PR (the folder is preserved in -history and in the upstream template). +Low — the folder is inert documentation/tooling once setup ran. The permanent +document edits only remove references that would otherwise become stale. +Rollback: revert this PR (the folder is preserved in history and in the +upstream template). ## Evidence @@ -101,11 +161,32 @@ function Invoke-SetupRemoval { return 1 } + if (-not (Test-AIMaintainerPrecondition)) { + Write-Error 'AI-maintainer identity verification failed — setup will not be removed.' + return 1 + } + git switch -c $BranchName if ($LASTEXITCODE -ne 0) { Write-Error "Could not create branch '$BranchName' (does it already exist?)." return 1 } + + try { + $updatedDocs = @(Remove-TemplateOnlyDocumentation) + } + catch { + Write-Error "Could not clean template-only documentation: $($_.Exception.Message)" + return 1 + } + if ($updatedDocs.Count -gt 0) { + git add -- @updatedDocs + if ($LASTEXITCODE -ne 0) { + Write-Error 'Staging permanent-document cleanup failed.' + return 1 + } + } + git rm -r -q -- setup if ($LASTEXITCODE -ne 0) { Write-Error 'git rm -r setup failed — is the working directory the repo root?' diff --git a/setup/MIGRATION.md b/setup/MIGRATION.md index 3589cd2..37eb653 100644 --- a/setup/MIGRATION.md +++ b/setup/MIGRATION.md @@ -11,25 +11,27 @@ first. [`CHANGELOG.md`](CHANGELOG.md) has the version history. This file lives in `setup/` alongside the one-time setup tooling — it's only needed during the migration, so it goes away with the rest of the folder when -`Complete-Setup.ps1` runs (step 12). +`Complete-Setup.ps1` runs (step 13). ## The concepts you're migrating -| # | Concept | Lives in | Doc | -| -- | ------------------- | ---------------------------------- | ------------------------------------------------------------------------ | -| 1 | Orchestration | `.pre-commit-config.yaml` (root) | [`/AGENTS.md`](../AGENTS.md) + inline comments | -| 2 | Configs | `.config/` | [`/.config/README.md`](../.config/README.md) | -| 3 | Linting & testing | `.config/scripts/` | [`/.config/scripts/README.md`](../.config/scripts/README.md) | -| 4 | Policy rules | `.config/PSScriptAnalyzerRules/` | [`/.config/scripts/README.md`](../.config/scripts/README.md) | -| 5 | Opt-in tooling | `.config/overlays/` | [`/.config/overlays/vale/README.md`](../.config/overlays/vale/README.md) | -| 6 | CI & automation | `.github/` | [`/.github/README.md`](../.github/README.md) | -| 7 | Editor integration | `.vscode/` | [`/.vscode/README.md`](../.vscode/README.md) | -| 8 | Documentation graph | Markdown docs + README indexes | [`/docs/knowledge-format.md`](../docs/knowledge-format.md) | -| 9 | AI delegation | `AGENTS.md`, `.claude/`, `.codex/` | [`/CLAUDE.md`](../CLAUDE.md) + tool-specific config | -| 10 | One-time repo setup | `setup/` (delete after use) | [`README.md`](README.md) | - -Root convention files ride along as-is and need no adaptation: -`.editorconfig`, `.gitattributes` (`eol=lf`), `.gitignore`, `.claudeignore`. +| # | Concept | Lives in | Doc | +| -- | ------------------- | ---------------------------------- | -------------------------------------------------------------- | +| 1 | Orchestration | `.pre-commit-config.yaml` (root) | [`/AGENTS.md`](../AGENTS.md) + inline comments | +| 2 | Configs | `.config/` | [`/.config/README.md`](../.config/README.md) | +| 3 | Linting & testing | `.config/scripts/` | [`/.config/scripts/README.md`](../.config/scripts/README.md) | +| 4 | Policy rules | `.config/PSScriptAnalyzerRules/` | [`/.config/scripts/README.md`](../.config/scripts/README.md) | +| 5 | Opt-in tooling | `.config/overlays/` | [`/.config/overlays/README.md`](../.config/overlays/README.md) | +| 6 | CI & automation | `.github/` | [`/.github/README.md`](../.github/README.md) | +| 7 | Editor integration | `.vscode/` | [`/.vscode/README.md`](../.vscode/README.md) | +| 8 | Documentation graph | Markdown/AsciiDoc + README indexes | [`/docs/knowledge-format.md`](../docs/knowledge-format.md) | +| 9 | AI delegation | `AGENTS.md`, `.claude/`, `.codex/` | [`/CLAUDE.md`](../CLAUDE.md) + tool-specific config | +| 10 | Optional AI skills | `setup/optional-skills/` | [`optional-skills/README.adoc`](optional-skills/README.adoc) | +| 11 | One-time repo setup | `setup/` (delete after use) | [`README.md`](README.md) | + +Root convention files provide defaults: `.editorconfig`, `.gitattributes` +(`eol=lf`), `.gitignore`, and `.claudeignore`. Copy them, then adapt ignore and +unignore rules to the target repository's languages and tracked editor files. ## Migration steps @@ -78,10 +80,11 @@ repo's root. *Done when:* no check exists twice, and the test lanes (`Invoke-TestLane.ps1 -Lane Fast`) run green or report "no tests". -5. **Bring opt-in tooling across as inert overlays** (concept 5) — Vale, - semgrep, or anything else the source repo added as "complete but not - wired." Copy the folder and its `precommit-hook.yaml` fragment; don't paste - the fragment into the root config unless you're activating it now. +5. **Bring opt-in tooling across as inert overlays** (concept 5) — start from + `.config/overlays/README.md`, then select Vale, Semgrep, or another relevant + overlay. Follow the selected overlay's structure; some carry a + `precommit-hook.yaml` fragment, while documentation-only decision records do + not. Do not activate a fragment unless the target needs it now. *Done when:* overlays exist but `pre-commit run --all-files` doesn't run them. @@ -107,9 +110,10 @@ repo's root. 8. **Adopt the documentation graph** (concept 8). Copy `docs/knowledge-format.md` and adapt its deliberately reduced OKF-inspired - conventions to the target. Keep normal Markdown links between documents, - use README files as directory indexes where progressive disclosure helps, - and cite external claims. Do not add mandatory frontmatter, reserved + conventions to the target. Treat Markdown and AsciiDoc as first-class: + use each format's native relative links/cross-references, use README files as + directory indexes where progressive disclosure helps, and cite external + claims. Do not add mandatory frontmatter, reserved `index.md`/`log.md` semantics, or an OKF conformance claim unless the target is intentionally becoming an exchangeable knowledge bundle. *Done when:* permanent docs are reachable from `AGENTS.md` or a linked @@ -120,13 +124,25 @@ repo's root. layers the target uses: `CLAUDE.md` and `.claude/agents/` for Claude Code; `.codex/config.toml` and `.codex/agents/` for Codex. Adapt model names and supported capabilities rather than copying them blindly. Keep - product-specific policy out of `AGENTS.md`: Claude imports the shared file, + shared principles in `AGENTS.md`, but keep product-only model names and + mechanics in their product entry points. Claude imports the shared file, while Codex reads it directly and adds its own `developer_instructions` - from `.codex/config.toml` in trusted projects. + from `.codex/config.toml` in trusted projects. Review + [`docs/token-saving.adoc`](../docs/token-saving.adoc) before making + mechanical commit delegation a default; it usually preserves parent context + rather than reducing total tokens. *Done when:* every retained product entry point loads the shared guidance, every referenced custom agent exists, and no unused product layer remains. -10. **Copy the remaining docs.** Bring over the per-folder READMEs, trimmed to +10. **Copy optional AI skills only when needed** (concept 10). Read + [`optional-skills/README.adoc`](optional-skills/README.adoc), install a + bundled skill only when the target agent has no equivalent, and copy the + complete skill directory with its runtime references. Copy the human TDD + knowledge base only when the target should retain that learning material. + *Done when:* each retained skill loads in the target agent and no duplicate + equivalent skill was installed. + +11. **Copy the remaining docs.** Bring over the per-folder READMEs, trimmed to the target repo's actual tools, and update them to reflect what you migrated. Purge phantoms: verify every referenced tool exists in the target repo (e.g. there is no `asciidoctor-lint` gem — use @@ -134,21 +150,22 @@ repo's root. lists a nonexistent tool is worse than no doc. *Done when:* every tool named in a doc can be invoked in the target repo. -11. **Copy and run the one-time setup tooling** (concept 10) after the first +12. **Copy and run the one-time setup tooling** (concept 11) after the first push and one CI run (so the `lint` check the ruleset requires actually exists): copy the whole `setup/` folder (including this file and - `optional-skills/` — install the bundled `develop-with-tdd` skill only if - no equivalent TDD skill exists), then follow + `optional-skills/` only if step 10 needs it), then follow [`README.md`](README.md) — protection, security settings, AI-maintainer identity, in that order. **If the target repo is owned by a personal/user account** (not an - organization), expect `merge_queue` to be rejected by GitHub — a platform - limitation, not a bug. `Protect-MainBranch.ps1` posts each rule as its own - ruleset for exactly this reason: the other four protections still land. + organization), expect `merge_queue` to be rejected by GitHub — a + [documented availability constraint](https://docs.github.com/en/enterprise-cloud@latest/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue), + not a bug. Use a GitHub App rather than a separate collaborator account's + fine-grained PAT. `Protect-MainBranch.ps1` posts each rule as its own + ruleset for exactly this reason: the other protections still land. *Done when:* the rulesets appear in repo settings and `Test-AIMaintainerIdentity.ps1` passes from the agent's shell. -12. **Finish.** Delete the reference copy (`.temp/template/`), then run +13. **Finish.** Delete the reference copy (`.temp/template/`), then run `pwsh -NoProfile -File setup/Complete-Setup.ps1` — it removes `setup/` and opens the removal PR (and refuses to run if the repo is itself a template repo). diff --git a/setup/New-AIMaintainerApp.ps1 b/setup/New-AIMaintainerApp.ps1 index 6969b67..01ab7fb 100644 --- a/setup/New-AIMaintainerApp.ps1 +++ b/setup/New-AIMaintainerApp.ps1 @@ -213,7 +213,8 @@ App created: $($app.slug) (App ID $($app.id)) Private key: $keyPath — move it to your secret manager; never commit it. Next steps: 1. Install it on THIS repo only: $($app.html_url)/installations/new - 2. Mint an installation token for the agent's shell (see setup/AI-Maintainer-Identity.adoc). + 2. Note the Installation ID, then mint a repository-scoped token: + `$env:GH_TOKEN = pwsh -NoProfile -File setup/New-AIMaintainerToken.ps1 -AppId $($app.id) -InstallationId -Repository $repo -PrivateKeyPath '$keyPath'` 3. Verify from that shell: pwsh -NoProfile -File setup/Test-AIMaintainerIdentity.ps1 "@ -InformationAction Continue return 0 diff --git a/setup/New-AIMaintainerToken.ps1 b/setup/New-AIMaintainerToken.ps1 new file mode 100644 index 0000000..aa26ac7 --- /dev/null +++ b/setup/New-AIMaintainerToken.ps1 @@ -0,0 +1,140 @@ +#Requires -Version 7.0 +<# +.SYNOPSIS + Mint a repository-scoped GitHub App installation token for an AI agent. + +.DESCRIPTION + Signs a short-lived RS256 JSON Web Token with the App private key, then + exchanges it for an installation token scoped to one repository. The + installation must already grant that repository and the App permissions + established by New-AIMaintainerApp.ps1. + + The token is written to stdout only. Capture it into GH_TOKEN in the + dedicated agent shell; never write it to disk or pass it as a command-line + argument. +#> +[CmdletBinding()] +param( + [long]$AppId, + [long]$InstallationId, + [string]$Repository, + [string]$PrivateKeyPath +) + +Set-StrictMode -Version Latest + +function ConvertTo-Base64Url { + param([Parameter(Mandatory)][byte[]]$Bytes) + + [Convert]::ToBase64String($Bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_') +} + +function ConvertFrom-Base64Url { + param([Parameter(Mandatory)][string]$Value) + + $base64 = $Value.Replace('-', '+').Replace('_', '/') + switch ($base64.Length % 4) { + 2 { $base64 += '==' } + 3 { $base64 += '=' } + } + , [Convert]::FromBase64String($base64) +} + +function Get-GitHubAppJwt { + param( + [Parameter(Mandatory)][long]$AppId, + [Parameter(Mandatory)][string]$PrivateKeyPath, + [DateTimeOffset]$Now = [DateTimeOffset]::UtcNow + ) + + if (-not (Test-Path -LiteralPath $PrivateKeyPath -PathType Leaf)) { + throw "GitHub App private key not found: $PrivateKeyPath" + } + + $header = '{"alg":"RS256","typ":"JWT"}' + $payload = [ordered]@{ + iat = $Now.AddSeconds(-60).ToUnixTimeSeconds() + exp = $Now.AddMinutes(9).ToUnixTimeSeconds() + iss = "$AppId" + } | ConvertTo-Json -Compress + + $headerPart = ConvertTo-Base64Url -Bytes ([System.Text.Encoding]::UTF8.GetBytes($header)) + $payloadPart = ConvertTo-Base64Url -Bytes ([System.Text.Encoding]::UTF8.GetBytes($payload)) + $unsigned = "$headerPart.$payloadPart" + + $rsa = [System.Security.Cryptography.RSA]::Create() + try { + $rsa.ImportFromPem([System.IO.File]::ReadAllText($PrivateKeyPath)) + $signature = $rsa.SignData( + [System.Text.Encoding]::UTF8.GetBytes($unsigned), + [System.Security.Cryptography.HashAlgorithmName]::SHA256, + [System.Security.Cryptography.RSASignaturePadding]::Pkcs1 + ) + } + finally { + $rsa.Dispose() + } + + "$unsigned.$(ConvertTo-Base64Url -Bytes $signature)" +} + +function Get-InstallationAccessToken { + param( + [Parameter(Mandatory)][long]$AppId, + [Parameter(Mandatory)][long]$InstallationId, + [Parameter(Mandatory)][ValidatePattern('^[^/]+/[^/]+$')][string]$Repository, + [Parameter(Mandatory)][string]$PrivateKeyPath + ) + + $jwt = Get-GitHubAppJwt -AppId $AppId -PrivateKeyPath $PrivateKeyPath + $repositoryName = ($Repository -split '/', 2)[1] + $response = gh api --method POST ` + -H 'Accept: application/vnd.github+json' ` + -H "Authorization: Bearer $jwt" ` + -H 'X-GitHub-Api-Version: 2022-11-28' ` + "app/installations/$InstallationId/access_tokens" ` + -f "repositories[]=$repositoryName" 2>$null + + if ($LASTEXITCODE -ne 0 -or -not $response) { + throw 'GitHub did not create an installation access token. Check the App ID, installation ID, private key, and repository installation.' + } + + $token = ($response | ConvertFrom-Json).token + if (-not $token) { + throw 'GitHub returned an installation-token response without a token.' + } + $token +} + +function Invoke-AIMaintainerTokenCreation { + param( + [long]$AppId, + [long]$InstallationId, + [string]$Repository, + [string]$PrivateKeyPath + ) + + if (-not (Get-Command gh -ErrorAction SilentlyContinue)) { + throw 'GitHub CLI (gh) is not installed. https://cli.github.com/' + } + if ($AppId -le 0 -or $InstallationId -le 0 -or + [string]::IsNullOrWhiteSpace($Repository) -or + [string]::IsNullOrWhiteSpace($PrivateKeyPath)) { + throw 'AppId, InstallationId, Repository (owner/name), and PrivateKeyPath are required.' + } + + Get-InstallationAccessToken -AppId $AppId -InstallationId $InstallationId ` + -Repository $Repository -PrivateKeyPath $PrivateKeyPath +} + +if ($MyInvocation.InvocationName -ne '.') { + try { + Invoke-AIMaintainerTokenCreation -AppId $AppId -InstallationId $InstallationId ` + -Repository $Repository -PrivateKeyPath $PrivateKeyPath + exit 0 + } + catch { + Write-Error $_ + exit 1 + } +} diff --git a/setup/README.md b/setup/README.md index 85683ca..382fb07 100644 --- a/setup/README.md +++ b/setup/README.md @@ -23,6 +23,7 @@ pwsh -NoProfile -File setup/Enable-RepoSecurity.ps1 # 3. Set up the AI-maintainer identity, then verify it FROM THE AGENT'S SHELL pwsh -NoProfile -File setup/New-AIMaintainerApp.ps1 +# After installation, mint a repository-scoped token as shown by the script. pwsh -NoProfile -File setup/Test-AIMaintainerIdentity.ps1 # Optional: install the bundled AI skills before they are deleted with setup/ @@ -56,6 +57,7 @@ private-with-GitHub-Enterprise-Cloud) or Enterprise Cloud generally — GitHub rejects it outright on a personal/user-owned repo. The script warns and skips it in that case; the other four protections (no force-push, no deletion, required PR, required `lint` check) still land. +[GitHub documents merge-queue availability and setup requirements](https://docs.github.com/en/enterprise-cloud@latest/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue). ## 2. Enable server-side security settings @@ -76,26 +78,30 @@ set warn and are skipped; the script still applies the rest. Agents must work under their own least-privilege identity, never under your admin account. [`AI-Maintainer-Identity.adoc`](AI-Maintainer-Identity.adoc) is -the knowledge base; two scripts automate what can be automated: +the knowledge base; three scripts automate what can be automated: ```pwsh -# Create a least-privilege GitHub App (one browser confirmation — GitHub has -# no pure-API path for App creation). One App per locally-run agent. +# Create a least-privilege GitHub App through GitHub's manifest flow. +# One App per locally-run agent. pwsh -NoProfile -File setup/New-AIMaintainerApp.ps1 -# From the AGENT's shell: verify its credential is a bot identity +# Install the App on this repository, then use the command printed by the +# script to set GH_TOKEN with New-AIMaintainerToken.ps1. + +# From the AGENT's shell: verify its credential is a safe identity # (installation token or fine-grained PAT) WITH repo write/push and WITHOUT # admin. Fails closed. pwsh -NoProfile -File setup/Test-AIMaintainerIdentity.ps1 ``` Create Apps only for agents *you* run locally (Claude Code, Codex CLI, …). -Vendor-hosted agents — Copilot coding agent, Codex cloud — bring their own -GitHub App: install and repo-scope *theirs*; you can't substitute your own. +Vendor-hosted agents use their product's supported source-control integration; +scope it to only the intended repositories. ## Optional: install the bundled AI skills -[`optional-skills/`](optional-skills/) bundles two skills: `develop-with-tdd` +[`optional-skills/README.adoc`](optional-skills/README.adoc) indexes two +skills: `develop-with-tdd` (plus the TDD knowledge base behind it) that this repo's docs reference, and `changelog-entry` for adding high-quality `CHANGELOG.md` entries. If your agent already has an equivalent, skip it; no need for a second copy. Otherwise diff --git a/setup/Test-AIMaintainerIdentity.ps1 b/setup/Test-AIMaintainerIdentity.ps1 index f9640e6..1e1f9bd 100644 --- a/setup/Test-AIMaintainerIdentity.ps1 +++ b/setup/Test-AIMaintainerIdentity.ps1 @@ -50,7 +50,9 @@ function Get-IdentityVerdict { param( [Parameter(Mandatory)][string]$TokenKind, [Parameter(Mandatory)][bool]$HasAdmin, - [Parameter(Mandatory)][bool]$HasWrite + [Parameter(Mandatory)][bool]$HasWrite, + [string]$ActorLogin = '', + [string]$RepositoryOwner = '' ) $findings = [System.Collections.Generic.List[string]]::new() @@ -62,7 +64,15 @@ function Get-IdentityVerdict { } switch ($TokenKind) { 'Installation' { } - 'FineGrainedPat' { } + 'FineGrainedPat' { + if ([string]::IsNullOrWhiteSpace($ActorLogin) -or + [string]::IsNullOrWhiteSpace($RepositoryOwner)) { + $findings.Add('Could not verify that the fine-grained PAT belongs to a separate maintainer identity; failing closed.') + } + elseif ($ActorLogin.Equals($RepositoryOwner, [System.StringComparison]::OrdinalIgnoreCase)) { + $findings.Add('Fine-grained PAT belongs to the repository owner, not a separate maintainer identity. Use a GitHub App for a personal repository.') + } + } 'OAuthUser' { $findings.Add('Token is an OAuth user token — the owner''s interactive gh session, not a separate agent identity. Actions would be attributed to your own account.') } 'UserToServer' { $findings.Add('Token is a user-to-server App token — it acts as the signed-in user, so agent actions are attributed to you.') } 'ClassicPat' { $findings.Add('Token is a classic PAT — it cannot be scoped per-permission or per-repo. Replace it with a fine-grained PAT or an App installation token.') } @@ -91,6 +101,13 @@ function Invoke-IdentityCheck { return 1 } + $tokenKind = Get-TokenKind -Token $token + $repoOwner = ($repo -split '/', 2)[0] + $actorLogin = '' + if ($tokenKind -eq 'FineGrainedPat') { + $actorLogin = gh api user --jq '.login' 2>$null + } + # Effective permissions of THIS credential on THIS repo; missing/erroring # fields fail closed. $admin = gh api "repos/$repo" --jq '.permissions.admin' 2>$null @@ -98,7 +115,8 @@ function Invoke-IdentityCheck { $hasAdmin = "$admin" -eq 'true' $hasWrite = "$push" -eq 'true' - $verdict = Get-IdentityVerdict -TokenKind (Get-TokenKind -Token $token) -HasAdmin $hasAdmin -HasWrite $hasWrite + $verdict = Get-IdentityVerdict -TokenKind $tokenKind -HasAdmin $hasAdmin -HasWrite $hasWrite ` + -ActorLogin "$actorLogin" -RepositoryOwner $repoOwner if (-not $verdict.Pass) { $verdict.Findings | ForEach-Object { Write-Error $_ } Write-Error "This shell's credential is NOT a least-privilege AI-maintainer identity for $repo. See setup/AI-Maintainer-Identity.adoc." diff --git a/setup/optional-skills/README.adoc b/setup/optional-skills/README.adoc new file mode 100644 index 0000000..1ee9f75 --- /dev/null +++ b/setup/optional-skills/README.adoc @@ -0,0 +1,24 @@ += Optional AI skills + +This directory bundles optional, portable agent skills and their supporting +human knowledge. It is deleted with the rest of `setup/`, so install any wanted +skill before running `Complete-Setup.ps1`. + +== Installable skills + +* link:skills/develop-with-tdd/SKILL.md[develop-with-tdd] — a compact runtime + workflow for trustworthy Red-Green-Refactor development. +* link:skills/changelog-entry/SKILL.md[changelog-entry] — guidance for useful + changelog entries. + +Copy a complete skill directory, including its references and metadata, into +the skill directory supported by the target agent. Skip installation when an +equivalent skill is already available. + +== Human knowledge + +xref:knowledge/tdd/README.adoc[The TDD knowledge base] is the human learning, +research, maintenance, and evaluation companion to the compact runtime skill. +It is written in AsciiDoc for first-class tables and cross-references; the +runtime package remains Markdown because agent skill entry points are named +`SKILL.md`. diff --git a/setup/optional-skills/knowledge/tdd/ASSESSMENT_BRIEF.adoc b/setup/optional-skills/knowledge/tdd/ASSESSMENT_BRIEF.adoc new file mode 100644 index 0000000..f7f2695 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/ASSESSMENT_BRIEF.adoc @@ -0,0 +1,138 @@ += TDD documentation and skill assessment brief + +Use this brief when asking an AI agent to review the TDD knowledge base and the +`develop-with-tdd` skill. + +The goal is not a generic writing review. The goal is to determine whether the +guide teaches the intended TDD practice clearly, and whether the skill applies +that practice reliably while changing code. + +== Review targets + +Assess these targets in this order: + +[arabic] +. xref:guide/01-why-tdd.adoc[The human guide], beginning with Chapter 1 +. xref:research-findings.adoc[Research findings] +. xref:traceability.adoc[Traceability] +. link:../../skills/develop-with-tdd/SKILL.md[The develop-with-tdd skill] + +Treat the guide as the human teaching authority. + +Treat the skill as an execution artifact that should operationalize the same +doctrine without needing to load the full guide at runtime. + +== Intended audience + +The human guide must work for: + +* beginners who do not yet know why TDD exists; +* practitioners who have used tests but may not have used TDD deliberately; +* skeptics who associate TDD with dogma, slow tests, brittle mocks, or coverage +theater; +* developers working in both greenfield and legacy code. + +Do not assess the guide as if every reader already understands test levels, +mocks, characterization tests, BDD, CI, or refactoring constraints. + +== Core doctrine to preserve + +The material should teach TDD as a fast-feedback design discipline, not as a +coverage ritual. + +The central rule is: + +____ +Prefer the shortest trustworthy feedback loop that faithfully exercises the +current risk. +____ + +The review should preserve these ideas: + +* coverage is a by-product and diagnostic signal, not the primary goal; +* the first value of TDD is thinking about desired behavior before +implementation; +* a TDD cycle clarifies a contract, produces a small design decision, and leaves +executable regression evidence behind; +* feedback speed includes relevance, fidelity, repeatability, diagnostic value, +and decision timing, not only runtime; +* test level and execution cadence are related but independent; +* a slower faithful test can be better than a fast irrelevant test; +* BDD, ATDD, and Specification by Example are related practices, not simple +replacements or synonyms for TDD; +* legacy work may need characterization, minimal behavior-preserving seams, and +smaller entry strategies before normal TDD becomes possible. + +== Required review questions + +Answer these questions with concrete file or section references: + +[arabic] +. Does the guide have a clear learning journey from "why TDD" to "how to +practise it" to "how to adapt it without dogma"? +. Are important terms introduced before the text relies on them? +. Does the guide explain why Red matters, including why an AI agent must not +infer that a test would fail? +. Does the guide distinguish Red, Green, Refactor, characterization, pure +refactoring, exploration, and intentional contract changes? +. Does it explain feedback lanes clearly enough for a team to choose fast, +standard, and thorough suites? +. Does it cover focused, integration, contract, acceptance, and E2E evidence +without implying one universal test pyramid? +. Does it address common misconceptions and failure modes without turning into +a defensive rant? +. Does it help both greenfield and legacy development? +. Does the skill preserve the same invariants as the guide? +. Are any examples misleading, incomplete, over-fitted, or too abstract for a +beginner? + +== Skill-specific checks + +When reviewing `skills/develop-with-tdd/`, check whether it instructs an AI +agent to: + +* establish a known baseline before attributing failures; +* choose the shortest trustworthy evidence for the current risk; +* write or revise evidence before implementing new or corrected behavior; +* run and observe a meaningful Red instead of assuming Red; +* distinguish harness, compile, environment, and behavior failures; +* reach Green without weakening the test or changing the agreed contract; +* refactor only under passing evidence; +* report what was run, what was omitted, and what uncertainty remains; +* use nonstandard branches honestly, such as characterization-first legacy work, +pure refactor, exploration without an oracle, or intentional contract change. + +Also check whether references are loaded conditionally. The skill should stay +compact by default and use deeper references only when the task needs them. + +== Output format for an AI reviewer + +Use this structure: + +[arabic] +. Findings by severity +. Missing or underdeveloped angles +. Skill and guide synchronization issues +. Suggested concrete edits +. Residual risks or open questions + +For each finding, include: + +* location; +* why it matters; +* suggested correction. + +Avoid vague praise. A clean assessment should still mention residual risks, +scope limits, and the strongest remaining objections. + +== Non-goals + +Do not require the guide to become a complete testing textbook. + +Do not add every adjacent practice into the core learning path. Security, +accessibility, performance, compliance, observability, mutation testing, and +formal methods belong in adjacent or appendix material unless they directly +change the TDD practice being taught. + +Do not optimize the skill for maximum coverage. Optimize it for trustworthy, +timely evidence while preserving the Red-Green-Refactor discipline. diff --git a/setup/optional-skills/knowledge/tdd/ASSESSMENT_BRIEF.md b/setup/optional-skills/knowledge/tdd/ASSESSMENT_BRIEF.md deleted file mode 100644 index 300bfac..0000000 --- a/setup/optional-skills/knowledge/tdd/ASSESSMENT_BRIEF.md +++ /dev/null @@ -1,133 +0,0 @@ -# TDD documentation and skill assessment brief - -Use this brief when asking an AI agent to review the TDD knowledge base and the -`develop-with-tdd` skill. - -The goal is not a generic writing review. The goal is to determine whether the -guide teaches the intended TDD practice clearly, and whether the skill applies -that practice reliably while changing code. - -## Review targets - -Assess these targets in this order: - -1. `knowledge/tdd/guide/` -2. `knowledge/tdd/research-findings.md` -3. `knowledge/tdd/traceability.md` -4. `skills/develop-with-tdd/` - -Treat the guide as the human teaching authority. - -Treat the skill as an execution artifact that should operationalize the same -doctrine without needing to load the full guide at runtime. - -## Intended audience - -The human guide must work for: - -- beginners who do not yet know why TDD exists; -- practitioners who have used tests but may not have used TDD deliberately; -- skeptics who associate TDD with dogma, slow tests, brittle mocks, or coverage - theater; -- developers working in both greenfield and legacy code. - -Do not assess the guide as if every reader already understands test levels, -mocks, characterization tests, BDD, CI, or refactoring constraints. - -## Core doctrine to preserve - -The material should teach TDD as a fast-feedback design discipline, not as a -coverage ritual. - -The central rule is: - -> Prefer the shortest trustworthy feedback loop that faithfully exercises the -> current risk. - -The review should preserve these ideas: - -- coverage is a by-product and diagnostic signal, not the primary goal; -- the first value of TDD is thinking about desired behavior before - implementation; -- a TDD cycle clarifies a contract, produces a small design decision, and leaves - executable regression evidence behind; -- feedback speed includes relevance, fidelity, repeatability, diagnostic value, - and decision timing, not only runtime; -- test level and execution cadence are related but independent; -- a slower faithful test can be better than a fast irrelevant test; -- BDD, ATDD, and Specification by Example are related practices, not simple - replacements or synonyms for TDD; -- legacy work may need characterization, minimal behavior-preserving seams, and - smaller entry strategies before normal TDD becomes possible. - -## Required review questions - -Answer these questions with concrete file or section references: - -1. Does the guide have a clear learning journey from "why TDD" to "how to - practise it" to "how to adapt it without dogma"? -2. Are important terms introduced before the text relies on them? -3. Does the guide explain why Red matters, including why an AI agent must not - infer that a test would fail? -4. Does the guide distinguish Red, Green, Refactor, characterization, pure - refactoring, exploration, and intentional contract changes? -5. Does it explain feedback lanes clearly enough for a team to choose fast, - standard, and thorough suites? -6. Does it cover focused, integration, contract, acceptance, and E2E evidence - without implying one universal test pyramid? -7. Does it address common misconceptions and failure modes without turning into - a defensive rant? -8. Does it help both greenfield and legacy development? -9. Does the skill preserve the same invariants as the guide? -10. Are any examples misleading, incomplete, over-fitted, or too abstract for a - beginner? - -## Skill-specific checks - -When reviewing `skills/develop-with-tdd/`, check whether it instructs an AI -agent to: - -- establish a known baseline before attributing failures; -- choose the shortest trustworthy evidence for the current risk; -- write or revise evidence before implementing new or corrected behavior; -- run and observe a meaningful Red instead of assuming Red; -- distinguish harness, compile, environment, and behavior failures; -- reach Green without weakening the test or changing the agreed contract; -- refactor only under passing evidence; -- report what was run, what was omitted, and what uncertainty remains; -- use nonstandard branches honestly, such as characterization-first legacy work, - pure refactor, exploration without an oracle, or intentional contract change. - -Also check whether references are loaded conditionally. The skill should stay -compact by default and use deeper references only when the task needs them. - -## Output format for an AI reviewer - -Use this structure: - -1. Findings by severity -2. Missing or underdeveloped angles -3. Skill and guide synchronization issues -4. Suggested concrete edits -5. Residual risks or open questions - -For each finding, include: - -- location; -- why it matters; -- suggested correction. - -Avoid vague praise. A clean assessment should still mention residual risks, -scope limits, and the strongest remaining objections. - -## Non-goals - -Do not require the guide to become a complete testing textbook. - -Do not add every adjacent practice into the core learning path. Security, -accessibility, performance, compliance, observability, mutation testing, and -formal methods belong in adjacent or appendix material unless they directly -change the TDD practice being taught. - -Do not optimize the skill for maximum coverage. Optimize it for trustworthy, -timely evidence while preserving the Red-Green-Refactor discipline. diff --git a/setup/optional-skills/knowledge/tdd/MAINTENANCE.adoc b/setup/optional-skills/knowledge/tdd/MAINTENANCE.adoc new file mode 100644 index 0000000..5d8b1a3 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/MAINTENANCE.adoc @@ -0,0 +1,49 @@ += Maintenance model + +The knowledge base has three authorities with different jobs: + +[arabic] +. *Evidence authority:* xref:research-findings.adoc[research-findings.adoc] owns + sourced historical, empirical, and definitional findings. +. *Teaching authority:* the xref:guide/01-why-tdd.adoc[guide] owns explanations, + progression, examples, and reader-facing trade-offs. +. *Execution authority:* link:../../skills/develop-with-tdd/SKILL.md[the runtime + skill] and its bundled references own agent behavior. + +`analysis.adoc` records editorial judgments. `traceability.adoc` connects the authorities; it does not duplicate their full content. The original AsciiDoc draft remains provenance, not an active authority. + +== Change procedure + +[arabic] +. Record new source evidence or a corrected interpretation in `research-findings.adoc`. +. Update the affected row in `traceability.adoc`. +. Update human explanation where reader understanding changes. +. Update the skill only where runtime agent behavior changes. +. Forward-test every changed skill branch with a task that does not reveal the intended answer. +. Validate local links, skill metadata, language, feedback-lane commands, and matrix destinations. + +Record forward-test scenarios and results in `skill-evaluation.adoc`. Replace hypothetical scenarios with runnable fixtures when a future change depends on stack-specific execution rather than decision quality. + +Keep user-facing installation and learning guidance in the top-level +xref:README.adoc[README.adoc]. The installable skill contains only `SKILL.md`, +provider metadata, and runtime resources. + +== Scope rule + +Include a perspective when it materially changes at least one of: + +* what behavior or risk is tested; +* where the observation boundary belongs or whether a seam is needed; +* which oracle or test double is appropriate; +* how quickly and reliably feedback arrives; +* which feedback lane should run during the cycle or before a risky transition; +* whether another assurance technique is better; +* legal, regulatory, safety, or operational obligations. + +Record deliberately excluded detail in the relevant human chapter. Do not add a method merely to make the catalogue exhaustive. + +Place a topic in the adjacent-practices appendix when it helps a reader recognize TDD’s boundary but does not change the ordinary Red–Green–Refactor procedure. + +== Duplication rule + +Repeat a short core principle only when an audience must act without loading another artifact. Keep evidence in research, explanations in human chapters, and imperatives in the skill. Link instead of copying examples or source discussions into the skill. diff --git a/setup/optional-skills/knowledge/tdd/MAINTENANCE.md b/setup/optional-skills/knowledge/tdd/MAINTENANCE.md deleted file mode 100644 index 3db7729..0000000 --- a/setup/optional-skills/knowledge/tdd/MAINTENANCE.md +++ /dev/null @@ -1,42 +0,0 @@ -# Maintenance model - -The knowledge base has three authorities with different jobs: - -1. **Evidence authority:** `research-findings.md` owns sourced historical, empirical, and definitional findings. -2. **Teaching authority:** `human/**/*.md` owns explanations, progression, examples, and reader-facing trade-offs. -3. **Execution authority:** `develop-with-tdd/SKILL.md` and its bundled reference own agent behavior. - -`analysis.md` records editorial judgments. `traceability.md` connects the authorities; it does not duplicate their full content. The original AsciiDoc draft remains provenance, not an active authority. - -## Change procedure - -1. Record new source evidence or a corrected interpretation in `research-findings.md`. -2. Update the affected row in `traceability.md`. -3. Update human explanation where reader understanding changes. -4. Update the skill only where runtime agent behavior changes. -5. Forward-test every changed skill branch with a task that does not reveal the intended answer. -6. Validate local links, skill metadata, language, feedback-lane commands, and matrix destinations. - -Record forward-test scenarios and results in `skill-evaluation.md`. Replace hypothetical scenarios with runnable fixtures when a future change depends on stack-specific execution rather than decision quality. - -Keep user-facing installation and learning guidance in the top-level `README.md`. The installable skill contains only `SKILL.md`, provider metadata, and runtime resources. - -## Scope rule - -Include a perspective when it materially changes at least one of: - -- what behavior or risk is tested; -- where the observation boundary belongs or whether a seam is needed; -- which oracle or test double is appropriate; -- how quickly and reliably feedback arrives; -- which feedback lane should run during the cycle or before a risky transition; -- whether another assurance technique is better; -- legal, regulatory, safety, or operational obligations. - -Record deliberately excluded detail in the relevant human chapter. Do not add a method merely to make the catalogue exhaustive. - -Place a topic in the adjacent-practices appendix when it helps a reader recognize TDD’s boundary but does not change the ordinary Red–Green–Refactor procedure. - -## Duplication rule - -Repeat a short core principle only when an audience must act without loading another artifact. Keep evidence in research, explanations in human chapters, and imperatives in the skill. Link instead of copying examples or source discussions into the skill. diff --git a/setup/optional-skills/knowledge/tdd/README.adoc b/setup/optional-skills/knowledge/tdd/README.adoc new file mode 100644 index 0000000..51423c6 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/README.adoc @@ -0,0 +1,84 @@ += TDD knowledge base + +This directory separates a human learning path from the compact instructions an AI agent needs while changing code. + +The human chapters are the teaching authority. The `develop-with-tdd` folder is a self-contained execution package; it does not load the human curriculum during development. + +== Choose a route + +[cols=",,",options="header",] +|=== +|Reader or need |Start here |Continue with +|New to TDD |xref:guide/01-why-tdd.adoc[Why TDD exists] |Practise → misconceptions → first worked example +|Ready to try one cycle |xref:guide/02-practising-tdd.adoc[Practising TDD deliberately] |Greenfield example in Chapter 7 +|Skeptical of TDD |xref:guide/06-applying-tdd-without-dogma.adoc[Applying TDD without dogma] |Research findings and worked examples +|Changing legacy code |xref:guide/06-applying-tdd-without-dogma.adoc#starting-in-legacy-code[Legacy guidance] |Legacy worked example +|Choosing focused, integration, contract, or E2E evidence |xref:guide/04-choosing-evidence-and-feedback-lanes.adoc[Choosing evidence and feedback lanes] |Decision examples and adjacent practices +|Introducing TDD to a team |xref:guide/06-applying-tdd-without-dogma.adoc#introducing-tdd-to-a-team[Team adoption] |Feedback lanes and maintenance model +|Asking an AI agent to develop through TDD |link:../../skills/develop-with-tdd/SKILL.md[`develop-with-tdd`] |Installation notes below +|Asking an AI agent to assess the material |xref:ASSESSMENT_BRIEF.adoc[Assessment brief] |Guide → research → traceability → skill +|=== + +== Core learning path + +These chapters form the main argument and should be read in order by a beginner: + +[arabic] +. xref:guide/01-why-tdd.adoc[Why test-driven development exists] +. xref:guide/02-practising-tdd.adoc[Practising TDD deliberately] +. xref:guide/03-common-misconceptions-and-failure-modes.adoc[Common misconceptions and failure modes] +. xref:guide/04-choosing-evidence-and-feedback-lanes.adoc[Choosing evidence and feedback lanes] +. xref:guide/05-bdd-and-collaborative-discovery.adoc[BDD and collaborative discovery] + +== Applied guidance + +[arabic] +. xref:guide/06-applying-tdd-without-dogma.adoc[Applying TDD without dogma] +. xref:guide/07-worked-examples.adoc[Worked examples] + +== Reference and optional material + +* xref:guide/appendices/stack-specific-notes.adoc[Stack-specific notes] +* xref:guide/appendices/glossary.adoc[Glossary] +* xref:guide/appendices/adjacent-practices-and-further-reading.adoc[Adjacent practices and further reading] + +Optional material helps readers recognize when TDD needs another form of evidence. It is not prerequisite knowledge for attempting the core loop. + +== Install the AI skill + +The portable skill consists of `../../skills/develop-with-tdd/SKILL.md`, `../../skills/develop-with-tdd/references/`, and optional provider metadata in `../../skills/develop-with-tdd/agents/`. + +=== Claude Code + +Copy the complete `develop-with-tdd` folder to one of: + +* project scope: `.claude/skills/develop-with-tdd/`; +* global scope: `~/.claude/skills/develop-with-tdd/`. + +The directory name must match the `name` in `SKILL.md`. Invoke it with `/develop-with-tdd` or allow its description to trigger it. + +To make TDD the default for behavioral code, add a matching rule to `CLAUDE.md` at the same scope: + +____ +When implementing or correcting behavior in production code, use the `develop-with-tdd` skill to establish Red, reach Green, and refactor under evidence. +____ + +Keep throwaway diagnostics and exploratory spikes outside that blanket rule. + +=== OpenAI + +Install the complete folder in the applicable skills directory. The `agents/openai.yaml` file supplies OpenAI UI metadata. + +Invoke the skill explicitly with `$develop-with-tdd` or allow model invocation when the environment supports it. + +== Analysis and provenance + +* xref:research-findings.adoc[Primary-source research findings] +* xref:traceability.adoc[Claim-to-deliverable traceability] +* xref:MAINTENANCE.adoc[Maintenance model] +* xref:ASSESSMENT_BRIEF.adoc[Assessment brief for AI reviewers] +* xref:analysis.adoc[Editorial assessment of the superseded source] +* xref:external-skill-survey.adoc[External skill survey] +* xref:skill-evaluation.adoc[Runtime-skill evaluation evidence] + +Superseded source material may remain in the old working area as provenance. It is not active documentation for this package. diff --git a/setup/optional-skills/knowledge/tdd/README.md b/setup/optional-skills/knowledge/tdd/README.md deleted file mode 100644 index 1452196..0000000 --- a/setup/optional-skills/knowledge/tdd/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# TDD knowledge base - -This directory separates a human learning path from the compact instructions an AI agent needs while changing code. - -The human chapters are the teaching authority. The `develop-with-tdd` folder is a self-contained execution package; it does not load the human curriculum during development. - -## Choose a route - -| Reader or need | Start here | Continue with | -| -------------------------------------------------------- | ---------------------------------------------------------------------------------------- | -------------------------------------------------- | -| New to TDD | [Why TDD exists](guide/01-why-tdd.md) | Practise -> misconceptions -> first worked example | -| Ready to try one cycle | [Practising TDD deliberately](guide/02-practising-tdd.md) | Greenfield example in Chapter 7 | -| Skeptical of TDD | [Applying TDD without dogma](guide/06-applying-tdd-without-dogma.md) | Research findings and worked examples | -| Changing legacy code | [Legacy guidance](guide/06-applying-tdd-without-dogma.md#starting-in-legacy-code) | Legacy worked example | -| Choosing focused, integration, contract, or E2E evidence | [Choosing evidence and feedback lanes](guide/04-choosing-evidence-and-feedback-lanes.md) | Decision examples and adjacent practices | -| Introducing TDD to a team | [Team adoption](guide/06-applying-tdd-without-dogma.md#introducing-tdd-to-a-team) | Feedback lanes and maintenance model | -| Asking an AI agent to develop through TDD | [`develop-with-tdd`](../../skills/develop-with-tdd/SKILL.md) | Installation notes below | -| Asking an AI agent to assess the material | [Assessment brief](ASSESSMENT_BRIEF.md) | Guide -> research -> traceability -> skill | - -## Core learning path - -These chapters form the main argument and should be read in order by a beginner: - -1. [Why test-driven development exists](guide/01-why-tdd.md) -2. [Practising TDD deliberately](guide/02-practising-tdd.md) -3. [Common misconceptions and failure modes](guide/03-common-misconceptions-and-failure-modes.md) -4. [Choosing evidence and feedback lanes](guide/04-choosing-evidence-and-feedback-lanes.md) -5. [BDD and collaborative discovery](guide/05-bdd-and-collaborative-discovery.md) - -## Applied guidance - -1. [Applying TDD without dogma](guide/06-applying-tdd-without-dogma.md) -2. [Worked examples](guide/07-worked-examples.md) - -## Reference and optional material - -- [Stack-specific notes](guide/appendices/stack-specific-notes.md) -- [Glossary](guide/appendices/glossary.md) -- [Adjacent practices and further reading](guide/appendices/adjacent-practices-and-further-reading.md) - -Optional material helps readers recognize when TDD needs another form of evidence. It is not prerequisite knowledge for attempting the core loop. - -## Install the AI skill - -The portable skill consists of `../../skills/develop-with-tdd/SKILL.md`, `../../skills/develop-with-tdd/references/`, and optional provider metadata in `../../skills/develop-with-tdd/agents/`. - -### Claude Code - -Copy the complete `develop-with-tdd` folder to one of: - -- project scope: `.claude/skills/develop-with-tdd/`; -- global scope: `~/.claude/skills/develop-with-tdd/`. - -The directory name must match the `name` in `SKILL.md`. Invoke it with `/develop-with-tdd` or allow its description to trigger it. - -To make TDD the default for behavioral code, add a matching rule to `CLAUDE.md` at the same scope: - -> When implementing or correcting behavior in production code, use the `develop-with-tdd` skill to establish Red, reach Green, and refactor under evidence. - -Keep throwaway diagnostics and exploratory spikes outside that blanket rule. - -### OpenAI - -Install the complete folder in the applicable skills directory. The `agents/openai.yaml` file supplies OpenAI UI metadata. - -Invoke the skill explicitly with `$develop-with-tdd` or allow model invocation when the environment supports it. - -## Analysis and provenance - -- [Primary-source research findings](research-findings.md) -- [Claim-to-deliverable traceability](traceability.md) -- [Maintenance model](MAINTENANCE.md) -- [Assessment brief for AI reviewers](ASSESSMENT_BRIEF.md) - -Superseded source material may remain in the old working area as provenance. It is not active documentation for this package. diff --git a/setup/optional-skills/knowledge/tdd/analysis.adoc b/setup/optional-skills/knowledge/tdd/analysis.adoc new file mode 100644 index 0000000..13f19af --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/analysis.adoc @@ -0,0 +1,226 @@ += Assessment of `testing-methodologies-foundation.adoc` + +== Executive assessment + +The draft contains unusually broad and useful material, but it is not yet a dependable foundation or a deployable AI instruction set. + +* *Cut:* not yet effective. A 1,800-line teaching text, four stack guides, a glossary, debate notes, and generated-skill source are mixed into one document. The AI section duplicates and sometimes hardens claims from the human section. +* *Completeness:* broad, but uneven. It covers more techniques than most TDD introductions, while leaving several important schools and decision criteria implicit. +* *Correctness:* the central thesis is defensible, but a number of categorical statements are opinions presented as definitions. Several examples also contradict the rules they are meant to teach. +* *Perspectives:* many are present, but the classicist/mockist distinction is described inconsistently, and the economic, sociotechnical, safety, and alternative-verification perspectives need explicit placement. + +[[1-is-the-document-sensibly-cut]] +== 1. Is the document sensibly cut? + +No. The current order works locally, but the document has four audiences and purposes: + +[arabic] +. a conceptual introduction for engineers; +. a technique catalogue; +. stack-specific reference material; +. operational instructions for an AI agent. + +Those purposes have different reading patterns. Human teaching needs a dependency order: purpose before loop, loop before test design, and test design before variants. Stack advice and agent rules are lookup material and interrupt that path. + +The embedded AI chapter is especially costly. It says it is the single source of truth, yet it derives some files from other sections and admits one external divergence. That is a generation specification, not an AI-optimized instruction set. + +The replacement structure preserves this assessment as provenance rather than +bundling the superseded original draft, and separates: + +* seven core and applied human chapters plus optional appendices; +* one condensed AI reference; +* one executable skill with progressively disclosed branches; +* one research record and this assessment. + +[[2-are-the-statements-complete-and-correct]] +== 2. Are the statements complete and correct? + +=== Claims to retain + +* TDD is a development discipline, not a coverage program. +* Coverage is evidence about execution, not proof of useful assertions or correct behavior. +* Writing an executable example before implementation can clarify a contract and expose interface friction. +* Small feedback loops and behavior-focused regression tests support safe change. +* Test level and test-first workflow are separate dimensions. +* BDD adds behavior language, outside-in framing, examples, and cross-role collaboration; Gherkin is optional. +* Unit, integration, contract, acceptance, E2E, property-based, mutation, and approval tests answer different questions. +* The empirical evidence for universal productivity or defect claims is mixed and context-dependent. + +=== Claims that need qualification or correction + +[arabic] +. *“Tests written after code are coupled to implementation.”* Timing increases anchoring risk but does not determine coupling. A post-hoc characterization or behavior test can be excellent; a test written first can still dictate internals. +. *“A verification test does not survive reimplementation.”* A black-box verification test can survive it. Durability follows from the observed contract, not from the label “verification.” +. *“The cost of changing the design is essentially zero.”* It is lower early, never zero. +. *“Tests make refactoring possible at all.”* Tests increase confidence; types, formal methods, review, observability, and disciplined manual checks can also support refactoring. +. *“BDD was explicitly created to correct the equation TDD = unit testing/coverage chasing.”* BDD did respond to recurring TDD adoption and language problems, but the draft assigns one narrow causal story too confidently. BDD later expanded into discovery, examples, ubiquitous language, and collaboration. +. *“BDD is not a separate methodology” / “BDD and ATDD are effectively synonymous.”* These are useful simplifications only if labeled as such. Communities use the terms differently: ATDD centers acceptance criteria; BDD also includes discovery and domain language. +. *“TDD drives E2E tests just as well as unit tests.”* Test-first can operate at any level, but slow or fragile system tests cannot provide the same tight design loop. Double-loop TDD or acceptance-test-driven development is the more precise model. +. *“Red means assertion failure, not compilation failure.”* This is a strong local discipline for proving test sensitivity, not the universal definition of red. Other established formulations count compile or missing-symbol failures. +. *Fake It example:* replacing `return 5` with `return a {plus} b` under one example changes behavior for other inputs. Calling that change a refactor contradicts behavior-preserving refactoring. Add a contrasting example to justify generalization, or call it an implementation step. +. *A test with no explicit assertion has zero verification.* It still verifies that setup and execution do not fail, and some frameworks encode assertions through interactions or properties. The right warning is that “does not throw” is often too weak unless that is the intended contract. +. *“The function could raise and the no-assertion test would pass.”* An uncaught exception fails ordinary test runners. +. *Mutation testing “measures test quality.”* Mutation score is a useful fault-detection proxy with equivalent-mutant and tooling limitations; it is not a complete quality measure. +. *Classicist vs. London school:* the draft later associates the London school with “never mock what you own.” London/mockist TDD commonly mocks collaborators, including owned collaborators, to drive object interactions. The text currently contradicts itself. +. *“One When per scenario” and “one concept per test.”* Good heuristics, not definitions. Workflow scenarios may legitimately contain a sequence; optimize for one coherent behavior and diagnostic clarity. +. *Pipeline cadence:* “acceptance/integration only at merge time or on demand” is too broad. Run any deterministic, sufficiently fast, valuable test as early as practical; separate suites by feedback cost rather than label. +. *SQLite as a database substitute:* even in-memory SQLite differs from production engines. Use it only when those semantic differences are irrelevant; otherwise use the production engine. +. *Tool prescriptions:* “always pytest,” “always FluentAssertions,” or “never mock a database” should defer to repository conventions and the risk being tested. +. *Runtime artifacts ship no tests:* packaging policy is project-specific and not part of TDD. + +=== The user’s data-transformation heuristic + +“Every function that transforms data is a candidate” is a good discovery heuristic, not a test rule. Prioritize a transformation when at least one of these applies: + +* its contract contains non-trivial rules or invariants; +* incorrect output has meaningful impact; +* inputs contain important partitions or edge cases; +* the code changes often or is hard to reason about; +* a focused test gives faster diagnosis than a broader test. + +Trivial mappings already covered through a stable public behavior may not deserve a separate test. Conversely, orchestration, authorization, retries, concurrency, migrations, and integration contracts can deserve tests even when they do not look like data transformations. + +[[3-are-all-viewpoints-at-least-mentioned]] +== 3. Are all viewpoints at least mentioned? + +No finite document can include every school, but a foundation should orient the reader to the major axes below. + +[cols=",,",options="header",] +|=== +|Axis |Present in draft |Needed treatment +|Detroit/classicist vs. London/mockist |Partial and inconsistent |Correct definitions and state trade-offs without declaring one universal winner +|Inside-out vs. outside-in |Mostly through BDD |Name both design directions explicitly +|Sociable vs. solitary unit tests |Implicit |Connect to collaborator choice and test doubles +|Unit TDD vs. double-loop/acceptance TDD |Implicit nested loops |Name double-loop TDD and distinguish feedback speeds +|BDD vs. ATDD vs. Specification by Example |Collapsed together |Preserve overlap while explaining different emphases +|Pyramid vs. trophy |Present |Add the broader rule: shape the portfolio around risks and architecture +|Example-based vs. property/metamorphic/model-based testing |Property only |Mention metamorphic and model-based approaches for oracle-poor systems +|Coverage vs. mutation |Present |Add branch/condition coverage and mutation limitations +|Legacy characterization |Strong |Add the risk that characterization freezes accidental behavior +|Sociotechnical collaboration |Present through Three Amigos |Connect it to shared understanding, not only executable tests +|Empirical/economic view |Empirical section present |Add cost of maintenance, flakiness, feedback delay, and opportunity cost +|Alternative assurance |Mostly absent |Mention static analysis, types, review, formal methods, monitoring, and manual exploration +|Non-functional qualities |Scattered |Mention security, accessibility, performance, resilience, concurrency, and operability +|Data/ML/LLM systems |LLM section present |Distinguish deterministic pipeline tests, data checks, statistical evaluation, and online monitoring +|=== + +== Recommended thesis + +The strongest version of the document’s core idea is: + +____ +TDD is a feedback discipline for discovering and evolving observable contracts in small steps. The tests it leaves behind are valuable, but neither coverage nor test count is the objective. Test scope, doubles, and complementary techniques should be selected from the risk and feedback needed, not from a universal pyramid or a rule that every method must be tested. +____ + +The historical “stille Post” story should be presented as an interpretation of common organizational failure, not as the singular history of TDD or BDD. + +== Comparison with the existing global `tdd` skill + +The Matt Pocock skill is a useful compact baseline. Its strongest ideas are vertical slices, behavior through public interfaces, independent expected values, and explicit resistance to implementation-coupled tests. + +It should not be copied unchanged: + +* It requires user confirmation of every seam. That can interrupt routine implementation even when the repository already makes the seam clear. The new skill asks only when the choice would settle a material product or architecture decision. +* It says refactoring is not part of the loop. That is a deliberate workflow convention, but it conflicts with the standard red–green–refactor formulation and with the core understanding being documented here. The new skill retains refactoring and gives it its own completion criterion. +* It treats boundary-only mocking as a universal rule rather than a classicist default. The new reference names classicist and London/mockist styles accurately and requires a deliberate choice. +* It does not cover bug reproduction, legacy characterization, outer acceptance loops, probabilistic systems, or complementary evidence. The new skill adds those as reference branches while keeping the executable process short. + +The result is still a skill for an ordinary coding agent, not a specialized standalone agent. `agents/openai.yaml` is presentation metadata for the skill. + +== Second-order review: did the deliverables perform their jobs? + +The first review concentrated on factual accuracy, document boundaries, and missing schools of thought. That was necessary but insufficient. A correction is only complete when it reaches every artifact that depends on it. + +Seven additional evaluation dimensions therefore govern the second iteration: + +[arabic] +. *Traceability:* connect each material source claim to its research verdict and destination artifacts. +. *Pedagogy:* retain worked examples, explicit learning progression, and optional stack guidance instead of reducing the human version to a manifesto. +. *Rule strength:* distinguish Definition, Default, Heuristic, and Project decision in AI instructions. +. *Deployability:* keep every runtime dependency of the skill inside its package. +. *Behavioral evaluation:* test the skill against diverse realistic scenarios, not only frontmatter rules. +. *Maintenance authority:* declare which artifact owns evidence, explanation, and executable behavior. +. *Scope criteria:* include a perspective when it changes test selection, interpretation, or applicability; record deliberate omissions rather than silently losing them. + +=== Gaps discovered by the second-order review + +* Several research corrections had not reached the human and AI projections. +* The skill depended on an operating reference outside its installable directory. +* The condensed human version lost most examples and all stack-specific guidance. +* The human chapters contained few citations even where claims were historical or empirical. +* The same guidance appeared in multiple files without a documented maintenance authority. +* A single simple forward test did not exercise legacy, integration, mockist, nondeterministic, or outer-loop branches. + +=== Resolved inconsistencies + +* Outer acceptance examples may guide inner work, but must remain pending, branch-local, or excluded from required gates until an evaluated suite expects them to pass. +* A controlled internal interface may change during refactoring when callers change with it and behavior at the governing boundary remains stable. +* A mock proves that an external effect was requested at a boundary, not that payment, delivery, or another real-world effect occurred. +* Coverage can be a diagnostic or regulated objective without becoming a complete quality measure. +* The skill can apply to controlled nondeterminism when an executable property, threshold, model, or deterministic seam provides an oracle. + +The resulting maintenance and propagation rules live in `MAINTENANCE.adoc` and `traceability.adoc`. + +== Comparison with public testing skills + +The survey in `external-skill-survey.adoc` found no public skill with a stronger general foundation than this knowledge base. The best transferable ideas were practical rather than definitional. + +The skill now adopts contrastive examples, mutation-sensitive data selection, meaningful factories, test-level decision examples, contract lifecycle, E2E diagnostics, flake ownership, and gate placement. + +It deliberately rejects fixed test ratios, universal mocking policies, mandatory coverage or mutation targets, one-assertion rigidity, and framework-specific recipes in the core workflow. + +== Narrative architecture review + +The first Markdown projection corrected facts but did not preserve a learnable argument. It read as a set of accurate positions rather than a journey from motivation to practice. + +The main failure was dependency order. Terms such as test-first, test quality, artifact roles, Red–Green–Refactor, and coverage appeared before the reader had a concrete cycle or a reason to need each distinction. + +“Addressing beginners” cannot be isolated in one section. Beginner accessibility is a constraint on every section: ground concepts before use, motivate each transition, and carry concrete behavior through the explanation. + +The revised human path uses this dependency order: + +[arabic] +. define TDD briefly; +. establish the change-safety and requirement problem; +. distinguish test-first from test-after without ranking individual tests by timing; +. state benefits and limitations before making universal claims; +. demonstrate one complete cycle; +. explain deliberate practice, its invariants, and recovery from failed cycles; +. address common misconceptions before widening the vocabulary; +. separate TDD workflow from test scope, fidelity, and execution cadence; +. explain BDD and collaborative discovery as improvements to behavioral input; +. address applicability, objections, greenfield, legacy, adoption, and incentives; +. demonstrate complete cycles and retain broader material as optional reference. + +The original AsciiDoc remains the content quarry. Its worked reasoning on triangulation, pitfalls, BDD, doubles, objections, greenfield, and legacy adoption was preserved where research supported it. + +The rewrite changes presentation and dependency order rather than discarding the original thesis. Claims corrected by `research-findings.adoc` remain corrected in the new narrative. + +== Comprehensive coherence review + +The next review treated fast feedback as the organizing mechanism rather than one benefit among many. + +“Fast” now means the shortest trustworthy loop that faithfully exercises the current risk. Runtime alone is insufficient; relevance, fidelity, repeatability, diagnostic value, and decision timing determine feedback quality. + +The human and AI projections share a small invariant set: + +[arabic] +. know the relevant baseline; +. frame one behavioral decision, risk, observation boundary, and independent oracle; +. observe a relevant, reproducible, attributable Red for new or corrected behavior; +. reach Green through production behavior rather than weakened evidence; +. separate behavioral change from behavior-preserving refactoring; +. close the current cycle before starting the next; +. report verification and remaining uncertainty honestly. + +Characterization, pure refactoring, exploration, and work without a useful executable oracle are explicit branches rather than silent violations of the normal cycle. + +The curriculum now separates common misconceptions from serious objections. Misconceptions lead to recognizable failure modes and recovery actions; objections examine whether TDD’s mechanisms repay their costs. + +The former broader-testing chapter was split because it combined distinct questions: evidence scope and fidelity, BDD discovery, testability, portfolio heuristics, and alternative assurance. + +Fast, standard, and thorough feedback lanes now describe execution cadence. They are project agreements based on measured cost and risk, not intrinsic properties of test labels. + +Worked material is divided into complete development sequences and compact decision examples. This preserves breadth without presenting short recommendations as demonstrations of practice. + +Optional techniques now live in a curated adjacent-practices appendix. A topic belongs there when it helps readers recognize TDD’s boundary but does not change the ordinary Red–Green–Refactor procedure. diff --git a/setup/optional-skills/knowledge/tdd/external-skill-survey.adoc b/setup/optional-skills/knowledge/tdd/external-skill-survey.adoc new file mode 100644 index 0000000..c0fdf66 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/external-skill-survey.adoc @@ -0,0 +1,283 @@ += Survey of Public Testing Skills + +== Decision summary + +The public skills surveyed do not provide a better general TDD model than +`develop-with-tdd`. The local skill is more careful about risk, test level, +oracle quality, the classicist/mockist distinction, coverage, and the limits of +test doubles. + +Several external skills nevertheless add real value in two areas: + +[arabic] +. contrastive examples that let an agent recognize good and bad test design; +. operational guidance for integration, contract, and E2E suites, including +failure artifacts, flake control, provider states, and environment fidelity. + +Those ideas should be adapted into self-contained references. The external +skills should not become runtime dependencies. + +== Method + +This survey was performed on 2026-07-13. It inspected public `SKILL.md` files, +their source repositories, and their skills.sh catalog entries. Catalog install +counts and repository popularity are discovery signals, not evidence of +technical correctness. + +Searches covered TDD, unit testing, integration testing, contract testing, E2E +testing, testing strategy, and test design. `npx skills find` was attempted for +the same terms, but did not return before the command was terminated in the +restricted environment. The skills.sh web catalog and linked repositories were +therefore used for discovery and source inspection. + +The comparison baseline was the local link:../../skills/develop-with-tdd/SKILL.md[`develop-with-tdd` skill] +and its bundled link:../../skills/develop-with-tdd/references/operating-reference.md[`operating-reference.md`]. + +== Sources and findings + +=== Matt Pocock: `tdd` + +* Source: https://github.com/mattpocock/skills/tree/main/skills/engineering/tdd[SKILL.md and package] +* Supporting sources: https://github.com/mattpocock/skills/blob/main/skills/engineering/tdd/tests.md[good and bad tests], https://github.com/mattpocock/skills/blob/main/skills/engineering/tdd/mocking.md[mocking and testability] +* Catalog: https://skills.sh/mattpocock/skills/tdd[skills.sh entry] +* Catalog install command: `npx skills add https://github.com/mattpocock/skills --skill tdd` + +The strongest contribution is not a new TDD rule. It is the use of short, +contrastive code examples: observable outcome versus incidental call, public +API versus back-door inspection, and injected boundary versus direct +construction. The vertical one-test/one-implementation rhythm also agrees with +the local skill. + +The examples need contextual qualification. An interaction can be the intended +observable contract in London-style TDD. A database query can be the right +oracle when the database artifact is the contract. “Never mock owned code” is a +classicist preference, not a definition of good testing. Requiring user approval +for every seam would also make routine repository work unnecessarily blocking. + +*Verdict:* adapt the contrastive teaching form and boundary-design examples; +reject the universal mocking and seam-approval rules. + +=== CityPaul: `tdd` and adjacent testing skills + +* Source: https://github.com/citypaul/.dotfiles/blob/main/claude/.claude/skills/tdd/SKILL.md[`tdd/SKILL.md`] +* Repository overview: https://github.com/citypaul/.dotfiles[testing, mutation-testing, finding-seams, and characterization skills] +* Catalog: https://skills.sh/citypaul/dotfiles/tdd[skills.sh entry] +* Catalog install command: `npx skills add https://github.com/citypaul/dotfiles --skill tdd` + +The valuable idea is mutation-aware example selection. The skill explicitly +asks whether boundary values, boolean combinations, arithmetic identities, +optional values, collection operations, and side effects would allow plausible +mutants to survive. Its example showing why multiplication by `1` is a weak +choice is a compact lesson in discriminating test data. + +The repository also demonstrates useful progressive disclosure: the TDD loop, +test design, mutation testing, legacy characterization, seam finding, and +refactoring are separate skills. That separation is useful as an editorial +model, although this project requires the deliverable itself to remain +self-contained. + +The main skill is too dogmatic for reuse as written. It requires mutation +testing in every cycle, 100% coverage by default, production changes only after +a failing test, and commit-history evidence of every phase. These are expensive +project policies, not universal TDD definitions. They conflict with the local +skill's risk-based evidence model and its treatment of exploration, regulated +coverage, legacy work, and alternative assurance techniques. + +*Verdict:* adapt mutation-sensitive examples and the modular reference shape; +reject mandatory mutation runs, universal 100% coverage, and Git-history +ceremony. + +=== Asyraf Hussin: `testing-best-practices` + +* Source: https://github.com/AsyrafHussin/agent-skills/blob/main/skills/testing-best-practices/SKILL.md[`SKILL.md`] +* Supporting rule package: https://github.com/AsyrafHussin/agent-skills/tree/main/skills/testing-best-practices[skill directory] +* Catalog: https://skills.sh/asyrafhussin/agent-skills/testing-best-practices[skills.sh entry] +* Catalog install command: `npx skills add https://github.com/asyrafhussin/agent-skills --skill testing-best-practices` +* Source metadata: version 2.0.0 and MIT license are declared in `SKILL.md`. + +This skill contributes a useful review vocabulary: independence, shared mutable +state, cleanup, deterministic control, minimal and realistic test data, +factories/builders, specific assertions, diagnostic messages, unhappy paths, +and suite organization for feedback speed. Its progressive rule files are more +actionable than a single abstract paragraph. + +Some rules are too categorical or too stack-specific. “One assertion per test” +should mean one coherent behavioral conclusion, not one assertion call. A fixed +`<50ms` unit-test target is not portable. The code examples assume +TypeScript/Jest/Vitest, and “mock only at boundaries” again expresses one school +of TDD. + +*Verdict:* adapt test-data and diagnostic-quality examples; keep timing and +assertion-count advice as heuristics, not rules. + +=== Asyraf Hussin: `e2e-playwright-testing` + +* Source: https://github.com/AsyrafHussin/agent-skills/tree/main/skills/e2e-playwright-testing[skill directory and `SKILL.md`] +* Catalog: https://skills.sh/asyrafhussin/agent-skills/e2e-playwright-testing[skills.sh entry] +* Catalog install command: `npx skills add https://github.com/AsyrafHussin/agent-skills --skill e2e-playwright-testing` + +Its reusable value is operational: prefer user-facing semantic locators, use +framework-aware waiting instead of arbitrary sleeps, reuse authenticated state +carefully, avoid shared-database parallelism when isolation is absent, and turn +real flake causes into explicit rules. These practices make E2E failures less +brittle and more diagnosable. + +The detailed rules are Playwright and web-stack specific. Authentication-state +reuse can also conceal login regressions unless a separate login journey remains +covered. Serial execution protects a shared database but can hide a test-data +isolation problem rather than solve it. + +*Verdict:* adapt the reliability principles and show them in one browser +example; do not embed Playwright or Laravel/React recipes in the core workflow. + +=== BMAD Labs: `typescript-e2e-testing` + +* Source: https://github.com/bmad-labs/skills/tree/main/skills/typescript-e2e-testing[`skills/typescript-e2e-testing`] +* Repository metadata and install paths: https://github.com/bmad-labs/skills[BMAD skills repository] +* Catalog: https://skills.sh/bmad-labs/skills/typescript-e2e-testing[skills.sh entry] +* Catalog install command: `npx skills add https://github.com/bmad-labs/skills --skill typescript-e2e-testing` +* Source license: the repository declares MIT. + +This package usefully treats E2E work as several different activities: setup, +write, review, run, debug, and optimize. It also emphasizes real infrastructure +and Given–When–Then workflows rather than calling a mocked controller test +“end-to-end.” That distinction is worth preserving. + +Most implementation guidance is deliberately narrow: TypeScript/NestJS, +Docker, Kafka, PostgreSQL, MongoDB, and Redis. The local skill must instead infer +the system boundary and infrastructure from the repository. “Real +infrastructure” also needs qualification: production-equivalent semantics are +required only for risks that depend on them. + +*Verdict:* adapt the activity breakdown and environment-fidelity question; +reject the fixed stack and unconditional infrastructure cost. + +=== `contract-testing-builder` + +* Source repository: https://github.com/patricio0312rev/skills[patricio0312rev/skills] +* Catalog and inspectable skill text: https://skills.sh/patricio0312rev/skills/contract-testing-builder[skills.sh entry] +* Catalog install command: `npx skills add https://github.com/patricio0312rev/skills --skill contract-testing-builder` + +The useful contribution is an end-to-end consumer-driven contract example: +consumer interaction generation, provider verification against the real +provider interface, provider-state setup, publishing/versioning, and a release +gate. This demonstrates why a contract test is not merely an HTTP unit test. + +The skill sometimes groups consumer-driven contracts, OpenAPI conformance, JSON +Schema checks, and handwritten object comparisons under one label. Those forms +answer related but different questions. A schema check cannot prove that a +consumer's exercised interaction is supported, while a consumer contract does +not replace a deployed-path check. Its examples are also tied to TypeScript and +Pact. + +*Verdict:* adapt a technology-neutral comparison of schema conformance, +consumer-driven contracts, provider verification, and E2E; reject framework +setup from the core skill. + +=== `qa-testing-strategy` + +* Source repository: https://github.com/vasilyu1983/AI-Agents-public[vasilyu1983/AI-Agents-public] +* Catalog and inspectable skill summary: https://skills.sh/vasilyu1983/ai-agents-public/qa-testing-strategy[skills.sh entry] +* Catalog install command: `npx skills add https://github.com/vasilyu1983/ai-agents-public --skill qa-testing-strategy` + +This is the strongest source for the broader portfolio view requested here. It +selects the smallest effective layer from unit through integration, contract, +and E2E, and adds separate merge/deploy gates, failure artifacts, logs/traces, +ownership, quarantine policy, flake service levels, and suite-time budgets. + +Those are testing-strategy and suite-operations concerns, not steps that belong +in every Red–Green–Refactor cycle. They should therefore appear as conditional +guidance for choosing or maintaining a broader test, not as mandatory overhead +for a focused deterministic example. + +*Verdict:* adopt the risk-to-evidence and operability vocabulary; adapt it as +conditional broader-test guidance. + +=== `testing-strategies` + +* Source repository: https://github.com/supercent-io/skills-template[supercent-io/skills-template] +* Catalog and inspectable skill text: https://skills.sh/supercent-io/skills-template/testing-strategies[skills.sh entry] +* Catalog install command: `npx skills add https://github.com/supercent-io/skills-template --skill testing-strategies` + +This skill is useful as evidence of a common public pattern: explain a test +pyramid, prescribe unit/integration/E2E layers, and add CI, isolation, and flaky +test advice. Most of that is already covered more precisely by the local skill. + +Its fixed 70/20/10 ratio is not transferable between systems. A compiler, a +database adapter, a distributed workflow, and a browser application have +different risk distributions. Counts also say nothing about oracle strength, +environment fidelity, or feedback time. + +*Verdict:* reject fixed ratios; retain the local risk-based portfolio model. + +== Adopt, adapt, or reject + +[cols=",,",options="header",] +|=== +|External idea |Decision |Reason and destination +|Contrastive good/bad test pairs |*Adopt* |Adds fast pattern recognition. Put in a conditional test-design reference. +|Independent expected value versus tautological oracle |*Adopt* |Directly strengthens the local oracle rule. Show literals, properties, trusted models, and a bad implementation-derived expectation. +|Mutation-sensitive test data |*Adapt* |Use as an example-selection heuristic; do not require mutation tooling in every cycle. +|Inject clock, randomness, network, storage, or external service boundaries |*Adapt* |Explain the design payoff and the risk of indirection created only for a mock framework. +|Narrow operation-specific ports |*Adapt* |Helpful at unstable external boundaries; not every owned collaborator needs an interface. +|Test-data factories/builders and minimal overrides |*Adopt* |Adds maintainability guidance missing from the runtime reference. Preserve meaningful domain values. +|Semantic browser locators and framework-aware waits |*Adapt* |Strong E2E reliability lesson; present technology-neutral principle plus one Playwright illustration. +|Failure artifacts, logs/traces, ownership, quarantine, and repair |*Adopt* |Adds operational suite-health guidance currently only implied. +|Merge gate versus deploy/smoke gate |*Adopt* |Clarifies where slower or environment-dependent evidence belongs. +|Consumer interaction, provider state, provider verification, publication, release gate |*Adopt* |Gives contract testing a complete lifecycle instead of a one-line definition. +|Schema conformance equals consumer-driven contract testing |*Reject* |These verify different compatibility claims and should be contrasted. +|E2E means only browser automation |*Reject* |E2E is an external system path; the entry point may be UI, API, CLI, message, or protocol. +|Fixed test-pyramid ratios |*Reject* |Test portfolios should follow risk, fidelity, diagnostic value, feedback time, and maintenance cost. +|Always mock internal collaborators / never mock internal collaborators |*Reject* |Both erase the classicist/mockist choice already documented locally. +|One assertion call per test |*Reject* |One coherent behavioral conclusion may need several observations. +|Mandatory 100% coverage, mutation score, or per-cycle mutation run |*Reject* |These are project or regulatory decisions, not universal TDD requirements. +|Commit history must expose every Red and Green |*Reject* |Useful optional audit evidence, but not proof of oracle quality or a universal workflow requirement. +|=== + +== What should be added to the local deliverables + +The survey supports adding examples, but not expanding `SKILL.md` into a testing +encyclopedia. The high-value shape is progressive disclosure: + +[arabic] +. `references/test-design-examples.md` +* behavior-coupled versus structure-coupled assertions; +* independent oracle versus tautology; +* discriminating versus identity test data; +* requested interaction versus verified external effect; +* classicist and mockist versions of the same behavior; +* test-data factory with minimal, meaningful overrides. +. `references/test-level-examples.md` +* focused domain rule; +* component/API test through a stable public seam; +* database integration where dialect or transaction semantics matter; +* schema conformance versus consumer-driven contract plus provider verification; +* E2E critical path with user-facing observations and failure artifacts; +* a risk-to-level comparison showing why duplicate assertions at every layer add little value. +. `references/design-for-testability.md` +* difficult boundaries and enabling points; +* narrow ports, controlled clocks/randomness, and observable results; +* when a real dependency, fake, stub, spy, or mock is appropriate; +* warning signs of test-induced design damage. +. A short operational section for broader tests +* baseline-green and reproducible-Red checks; +* environment fidelity chosen from the risk; +* deterministic state setup and cleanup; +* artifacts sufficient to diagnose a remote CI failure; +* explicit quarantine owner, reason, expiry, and repair path; +* separate fast merge evidence from slower deploy or scheduled evidence. + +`SKILL.md` should load those references only when their branch applies. The +existing operating reference should remain the authority for definitions and +decision strength. + +== Final assessment + +No surveyed public skill exposes a missing foundational TDD school or test level +that invalidates the current deliverables. The remaining gap is practical: +agents need examples that connect a risk to a seam, oracle, level, environment, +and diagnostic strategy. + +The recommended additions therefore provide actual new value. Copying external +coverage targets, fixed pyramid ratios, universal mocking rules, or complete +framework recipes would not. diff --git a/setup/optional-skills/knowledge/tdd/guide/01-why-tdd.md b/setup/optional-skills/knowledge/tdd/guide/01-why-tdd.adoc similarity index 74% rename from setup/optional-skills/knowledge/tdd/guide/01-why-tdd.md rename to setup/optional-skills/knowledge/tdd/guide/01-why-tdd.adoc index 3740fc2..61b0fcd 100644 --- a/setup/optional-skills/knowledge/tdd/guide/01-why-tdd.md +++ b/setup/optional-skills/knowledge/tdd/guide/01-why-tdd.adoc @@ -1,24 +1,24 @@ -# Why test-driven development exists += Why test-driven development exists This chapter assumes that you can read a small automated test. It does not assume that you have practised test-driven development or agree that it is useful. -## TDD in one minute +== TDD in one minute Test-driven development (TDD) is a way to develop one observable behavior at a time. A behavior is a response that matters to a caller or user. Before implementing it, the developer expresses the intended response as an executable example and runs that example. -For new behavior or a defect correction, the example must show that the current system does not yet satisfy the intended behavior. This observed, relevant failure is **Red**. +For new behavior or a defect correction, the example must show that the current system does not yet satisfy the intended behavior. This observed, relevant failure is *Red*. -The developer then changes the production code until the example passes. This is **Green**. While the evidence remains Green, the developer may improve the structure without changing behavior. This is **Refactor**. +The developer then changes the production code until the example passes. This is *Green*. While the evidence remains Green, the developer may improve the structure without changing behavior. This is *Refactor*. -Together, these steps form **Red–Green–Refactor**. +Together, these steps form *Red–Green–Refactor*. The immediate value is feedback while requirements and design decisions are still cheap to change. The retained examples later provide evidence against regressions. TDD is therefore both a development workflow and a way to produce tests. Having automated tests does not mean that the code was developed through TDD. -## TDD strengthens a familiar way of working +== TDD strengthens a familiar way of working Many developers already work iteratively: write a small part of the code, run the application, inspect the result, and continue when it appears to work. @@ -30,17 +30,17 @@ The result may also disappear when the developer moves on. Unless the scenario i TDD strengthens this loop in three ways: -- state the expected behavior before implementing it; -- use the shortest trustworthy feedback loop that faithfully exercises the current risk; -- retain the example so it can be rerun after later changes. +* state the expected behavior before implementing it; +* use the shortest trustworthy feedback loop that faithfully exercises the current risk; +* retain the example so it can be rerun after later changes. Each new behavior is checked alongside previously recorded behavior. This reduces the chance that an attempted improvement silently damages something that already worked. -German has a useful word for that outcome: **Verschlimmbesserung**—an attempted improvement that makes something worse. +German has a useful word for that outcome: *Verschlimmbesserung*—an attempted improvement that makes something worse. Tests can expose such regressions only when they describe the affected behavior with a useful expectation. Incorrectly specified, unstable, or untested behavior can still regress. -## The problem begins before code is written +== The problem begins before code is written Most software changes sound simple: add a rule, correct a defect, or alter an existing behavior. @@ -48,54 +48,56 @@ The first risk is not syntax. It is building the wrong outcome, overlooking an i Consider the request “orders of €50 or more receive free shipping.” Before code is written, several questions appear: -- Does exactly €50 qualify? -- Is the threshold applied before or after discounts? -- Do taxes or shipping fees contribute? -- What happens in another currency? +* Does exactly €50 qualify? +* Is the threshold applied before or after discounts? +* Do taxes or shipping fees contribute? +* What happens in another currency? Code can answer these questions accidentally. The first implementation may silently choose one interpretation. Tests derived from that code may then repeat the same assumptions. TDD asks for a concrete expected outcome first. This does not guarantee the right requirement, but it makes the current interpretation visible and reviewable before implementation hides it. -## What TDD tries to improve +== What TDD tries to improve TDD aims to shorten several costly delays: -- the delay before an ambiguous requirement becomes visible; -- the delay before an incorrect implementation is detected; -- the delay before an awkward interface is experienced; -- the delay before a later change reveals a regression. +* the delay before an ambiguous requirement becomes visible; +* the delay before an incorrect implementation is detected; +* the delay before an awkward interface is experienced; +* the delay before a later change reveals a regression. It does this through several modest mechanisms: -- **Requirement clarity:** an example makes an expected outcome concrete. -- **Interface feedback:** using an interface before implementing it exposes awkward inputs, outputs, and responsibilities. -- **Localized feedback:** small changes make a new failure easier to attribute. -- **Incremental design:** each example can challenge the current design before more structure accumulates. -- **Change safety:** retained examples can reveal regressions during later modification and refactoring. -- **External memory:** the suite records cases that would otherwise depend on developer memory. +* *Requirement clarity:* an example makes an expected outcome concrete. +* *Interface feedback:* using an interface before implementing it exposes awkward inputs, outputs, and responsibilities. +* *Localized feedback:* small changes make a new failure easier to attribute. +* *Incremental design:* each example can challenge the current design before more structure accumulates. +* *Change safety:* retained examples can reveal regressions during later modification and refactoring. +* *External memory:* the suite records cases that would otherwise depend on developer memory. These are mechanisms, not guaranteed project outcomes. Their value depends on the problem, the examples, the chosen test boundary, and the discipline of the developer. -## Why test-first and test-after provide different feedback +== Why test-first and test-after provide different feedback A test written after implementation can still verify behavior, document a contract, and detect later regressions. It cannot influence design decisions already made. Existing code also anchors attention: the test may demonstrate what the implementation does instead of challenging what it should do. -That is the relevant difference between **test-first** and **test-after**. It concerns when feedback becomes available, not whether one test is automatically good and the other bad. +That is the relevant difference between *test-first* and *test-after*. It concerns when feedback becomes available, not whether one test is automatically good and the other bad. Test-first can still produce a weak test. It may assert private details or derive the expected result from the same flawed reasoning used in production. Test timing creates an opportunity for design feedback. Test quality determines whether that opportunity is used well. Chapter 2 develops the criteria for trustworthy evidence. -## Fast feedback means more than a fast test +== Fast feedback means more than a fast test TDD is built around fast feedback, but test runtime is only one part of feedback speed. The governing rule is: -> Prefer the shortest trustworthy feedback loop that faithfully exercises the current risk. +____ +Prefer the shortest trustworthy feedback loop that faithfully exercises the current risk. +____ A useful loop is quick enough to guide the next decision, relevant to the behavior, repeatable, and diagnostic when it fails. @@ -105,7 +107,7 @@ Start as narrowly as the risk permits, not as narrowly as the test framework per The chapter on choosing evidence and feedback lanes develops this trade-off. -## What TDD does not promise +== What TDD does not promise TDD does not prove that requirements are correct or complete. A precise example of the wrong requirement is still wrong. @@ -119,24 +121,25 @@ The defensible claim is narrower: TDD changes when certain questions are asked a Whether that feedback is worth its cost must be judged for the code and risk at hand. The chapter on applying TDD without dogma examines evidence, objections, and limits. -## One observed cycle +== One observed cycle Suppose the agreed rule is: an order qualifies for free shipping when its discounted subtotal is at least €50. -### Establish a known baseline +=== Establish a known baseline Run the relevant existing tests first. If they pass, a later failure can be attributed to the new example. If they do not, record the existing failures before changing anything. -### Red: demonstrate one missing behavior +=== Red: demonstrate one missing behavior The first example defines one boundary case: -```python +[source,python] +---- def test_fifty_euros_qualifies_for_free_shipping(): order = Order(discounted_subtotal=Decimal("50.00")) assert order.qualifies_for_free_shipping() is True -``` +---- Run the test. Do not infer that it would fail, even when the change seems obvious. @@ -146,24 +149,25 @@ If a missing method prevents the assertion from running, add only enough structu This requirement applies equally to human and AI developers. Neither confidence nor generated reasoning is evidence that the test can fail. -### Green: satisfy the demonstrated behavior +=== Green: satisfy the demonstrated behavior Make the example pass with the smallest coherent production change: -```python +[source,python] +---- class Order: def __init__(self, discounted_subtotal): self.discounted_subtotal = discounted_subtotal def qualifies_for_free_shipping(self): return self.discounted_subtotal >= Decimal("50.00") -``` +---- Run the same test and observe Green. Do not obtain Green by weakening the assertion, skipping the test, swallowing the failure, or silently changing the requested contract. “Smallest” does not mean deliberately poor code. It means avoiding unrelated features, speculative abstractions, and cases that no current example or clear requirement demands. -### Refactor: improve structure without changing behavior +=== Refactor: improve structure without changing behavior With the test passing, inspect production and test code. Improve names, remove duplication, or clarify responsibilities while rerunning the relevant tests. @@ -171,7 +175,7 @@ Perhaps the threshold deserves a named policy. Perhaps no refactoring is justifi An intentional behavior change is not refactoring. It begins another cycle with a new or revised expectation. -### Continue with an informative example +=== Continue with an informative example A useful next example might show that €49.99 does not qualify. It examines the other side of the boundary instead of repeating the same fact with another large value. @@ -179,7 +183,7 @@ Another example could clarify whether discounts apply before the threshold. Each Finish this cycle at Green before starting the next behavior. That keeps failures attributable to a small, recent change. -## Why coverage is not the goal +== Why coverage is not the goal Coverage reports which code a suite executed. It does not show that expectations were correct, assertions were meaningful, or important faults would be detected. @@ -193,21 +197,21 @@ TDD often produces coverage because each new behavior is exercised by its exampl Mutation testing can probe whether tests detect selected changes, but its score is also evidence to interpret rather than a universal target. -## What the retained tests become +== What the retained tests become During development, an example helps make one decision. Afterward, the retained test may serve three related purposes: -- **Specification:** it records behavior the system is expected to preserve. -- **Verification:** it supplies evidence that the current implementation satisfies that behavior. -- **Regression detection:** it warns when a later change violates the recorded behavior. +* *Specification:* it records behavior the system is expected to preserve. +* *Verification:* it supplies evidence that the current implementation satisfies that behavior. +* *Regression detection:* it warns when a later change violates the recorded behavior. Design feedback is different because it occurs while an interface or responsibility is still being formed. The same test may contribute to all these jobs at different times. The next chapter turns this first cycle into a deliberate practice and explains how to choose behavior, evidence, expected results, and useful next examples. -## Sources and further reading +== Sources and further reading -- Kent Beck, [TDD is Kanban for Code](https://newsletter.kentbeck.com/p/tdd-is-kanban-for-code) -- Martin Fowler, [Test Coverage](https://martinfowler.com/bliki/TestCoverage.html) -- Rafique and Mišić, [The Effects of Test-Driven Development on External Quality and Productivity](https://doi.org/10.1109/TSE.2012.28) -- Munir et al., [Considering rigor and relevance when evaluating test driven development](https://doi.org/10.1016/j.infsof.2014.01.001) +* Kent Beck, https://newsletter.kentbeck.com/p/tdd-is-kanban-for-code[TDD is Kanban for Code] +* Martin Fowler, https://martinfowler.com/bliki/TestCoverage.html[Test Coverage] +* Rafique and Mišić, https://doi.org/10.1109/TSE.2012.28[The Effects of Test-Driven Development on External Quality and Productivity] +* Munir et al., https://doi.org/10.1016/j.infsof.2014.01.001[Considering rigor and relevance when evaluating test driven development] diff --git a/setup/optional-skills/knowledge/tdd/guide/02-practising-tdd.md b/setup/optional-skills/knowledge/tdd/guide/02-practising-tdd.adoc similarity index 69% rename from setup/optional-skills/knowledge/tdd/guide/02-practising-tdd.md rename to setup/optional-skills/knowledge/tdd/guide/02-practising-tdd.adoc index 3e7c543..ca65196 100644 --- a/setup/optional-skills/knowledge/tdd/guide/02-practising-tdd.md +++ b/setup/optional-skills/knowledge/tdd/guide/02-practising-tdd.adoc @@ -1,10 +1,10 @@ -# Practising TDD deliberately += Practising TDD deliberately Chapter 1 showed one cycle. This chapter turns that outline into a repeatable practice. The difficult part is rarely writing an assertion. It is choosing the next behavior, obtaining faithful evidence, and taking a step small enough to learn from. -## Start from behavior, not code inventory +== Start from behavior, not code inventory Do not begin by listing functions that need tests. Begin with a behavior a caller needs or a rule the system must preserve. @@ -16,14 +16,14 @@ For example, accepting an order may return an identifier and record the same ide The useful limit is one behavioral decision per cycle. If a failure could mean that several unrelated rules are wrong, split the example. -## Make a provisional test list +== Make a provisional test list Before the first test, list a few examples that might clarify the behavior. For the shipping rule, the list could be: -- exactly €50 qualifies; -- €49.99 does not qualify; -- the discounted subtotal determines eligibility; -- an unsupported currency is rejected. +* exactly €50 qualifies; +* €49.99 does not qualify; +* the discounted subtotal determines eligibility; +* an unsupported currency is rejected. This is a thinking aid, not a commitment to write the whole suite first. The list will change as each completed cycle reveals more about the requirement and design. @@ -31,7 +31,7 @@ Choose one example that is important, uncertain, and small enough to complete. L Do not implement all listed tests before production code. Doing so commits to an imagined design and discards the feedback one completed cycle should provide to the next. -## Establish a known baseline +== Establish a known baseline Run the smallest relevant existing test set before adding the example. @@ -41,9 +41,9 @@ A pre-existing failure is not the Red for the new behavior. Either restore the b On a new project with no tests, the baseline may be a successful build and confirmation that the test runner starts cleanly. This prevents infrastructure problems from masquerading as Red. -## Choose where to observe the behavior +== Choose where to observe the behavior -A test needs a route for supplying inputs and observing outcomes. That route is its **observation boundary**. +A test needs a route for supplying inputs and observing outcomes. That route is its *observation boundary*. For the shipping rule, the boundary could be an `Order` method, a pricing component, an HTTP endpoint, or a complete browser checkout. @@ -53,26 +53,26 @@ A narrow boundary often gives faster, more diagnostic feedback. A broader bounda Do not equate “narrow” with “one method.” Several functions or objects may form one coherent unit of behavior. -In legacy-code literature, a **seam** is a place where behavior or a dependency can be varied without editing the code at that point. A seam can make a useful observation boundary controllable. +In legacy-code literature, a *seam* is a place where behavior or a dependency can be varied without editing the code at that point. A seam can make a useful observation boundary controllable. -## Derive the expected result independently +== Derive the expected result independently -The basis on which a test decides whether behavior is acceptable is its **oracle**. +The basis on which a test decides whether behavior is acceptable is its *oracle*. An oracle is weak when the test calculates its expected result with the same algorithm as production. The same mistake can then appear on both sides and the test will pass. Prefer evidence independent of the implementation: -- an agreed example from a requirement or domain expert; -- a known literal such as the €50 boundary; -- an invariant such as “sorting preserves all input elements”; -- a trusted reference implementation; -- a reviewed baseline for complex existing output; -- a relationship between executions when an exact value is unavailable. +* an agreed example from a requirement or domain expert; +* a known literal such as the €50 boundary; +* an invariant such as “sorting preserves all input elements”; +* a trusted reference implementation; +* a reviewed baseline for complex existing output; +* a relationship between executions when an exact value is unavailable. If no reliable oracle exists, exact example-based TDD may not fit the uncertain part of the work. The exception paths later in this chapter explain what to do instead. -## Observe a meaningful Red +== Observe a meaningful Red Write or revise the smallest example of behavior the system does not yet satisfy, then run it. @@ -82,17 +82,17 @@ Observed Red answers the first question about a new test: can it fail before the A meaningful Red is: -- **observed:** the test was actually executed; -- **reproducible:** the same state produces the same failure; -- **relevant:** the intended behavior caused the failure; -- **discriminating:** an important wrong implementation would be detected; -- **attributable:** the new example or changed expectation explains the failure. +* *observed:* the test was actually executed; +* *reproducible:* the same state produces the same failure; +* *relevant:* the intended behavior caused the failure; +* *discriminating:* an important wrong implementation would be detected; +* *attributable:* the new example or changed expectation explains the failure. A broken fixture, wrong import, unavailable environment, or unrelated defect is not a behavioral Red. Repair the route to the behavior, return to the known baseline, and run again. For a defect, reproduce the defect first. Seeing the regression test fail against the current code shows that the test reaches the problem it claims to protect. -### If the new test passes immediately +=== If the new test passes immediately Do not add production code merely to manufacture a cycle. Investigate why the behavior already appears Green. @@ -104,7 +104,7 @@ This sensitivity check does not recreate the design feedback of genuine test-fir If the behavior genuinely exists, retain the test only if its regression value justifies its cost. Then select a behavior that is actually missing. -### If the test fails for the wrong reason +=== If the test fails for the wrong reason Repair the test route or add only the skeletal production structure needed to reach the intended assertion. Do not implement the behavior yet. @@ -112,13 +112,13 @@ Re-establish the baseline and rerun until the failure reaches the intended rule. Do not count a sequence of infrastructure errors as progress through Red. It proves only that the behavior has not yet been observed. -### If the suite was already failing +=== If the suite was already failing Separate the known failure from the new evidence. Record its command and output, fix it when in scope, or use an agreed isolated baseline. Never present an inherited failure as if the new example exposed it. -## Make the smallest coherent Green +== Make the smallest coherent Green Change only enough production code to satisfy the current example while preserving existing behavior. @@ -128,9 +128,9 @@ Green must come from satisfying the intended contract. Do not weaken an assertio Three implementation strategies are useful: -- **Obvious Implementation:** write the general solution directly when it is genuinely clear. -- **Fake It:** use a narrow result when a working example will help reveal the next step. -- **Triangulation:** add a contrasting example whose difference creates pressure for a more general rule. +* *Obvious Implementation:* write the general solution directly when it is genuinely clear. +* *Fake It:* use a narrow result when a working example will help reveal the next step. +* *Triangulation:* add a contrasting example whose difference creates pressure for a more general rule. Fake It is a learning strategy, not a ritual. Hard-coding an answer when the correct implementation is obvious adds ceremony without insight. @@ -138,7 +138,7 @@ Triangulation is useful when one example permits several plausible rules. A seco “Smallest” means the smallest coherent change. It does not require deliberately unreadable code or a solution known to violate an established requirement. -## Refactor only under Green +== Refactor only under Green Green means the demonstrated behavior works. It does not mean the design is finished. @@ -152,25 +152,25 @@ An internal interface may change during refactoring when all callers can change Refactoring is not mandatory motion. If the code already communicates the current design well, continue to the next behavior. -## Select the next example for information +== Select the next example for information The next test should establish something the current suite does not. Useful choices include: -- the other side of a boundary; -- an empty, missing, or malformed input; -- a competing business rule; -- a failure observed in production; -- a state transition that should be rejected; -- a contrasting example that creates pressure to generalize; -- an invariant over a wider input space. +* the other side of a boundary; +* an empty, missing, or malformed input; +* a competing business rule; +* a failure observed in production; +* a state transition that should be rejected; +* a contrasting example that creates pressure to generalize; +* an invariant over a wider input space. Changing €60 to €70 rarely adds information if both values exercise the same rule. Prefer examples that distinguish plausible implementations. Finish the current cycle at Green before beginning another behavioral decision. Closed cycles keep failures attributable and make interruptions safer. -## Change an existing contract deliberately +== Change an existing contract deliberately Tests are not immutable. When the intended behavior changes, the executable specification must change with it. @@ -184,18 +184,18 @@ Sometimes backward compatibility requires adding a new behavior while preserving Do not rewrite a failing test merely because production code changed accidentally. Change a test only when the intended contract, evidence, or test design has deliberately changed. -## What deserves a focused test? +== What deserves a focused test? Treat testing as a risk decision, not a census of functions. Strong candidates include: -- business rules and data transformations with meaningful partitions; -- validation, authorization, money, identity, time, and state transitions; -- parsers, serializers, calculations, and boundary conversions; -- concurrency, retry, idempotency, and failure recovery; -- public contracts between independently changing components; -- code that changes often or is difficult to diagnose through broader evidence. +* business rules and data transformations with meaningful partitions; +* validation, authorization, money, identity, time, and state transitions; +* parsers, serializers, calculations, and boundary conversions; +* concurrency, retry, idempotency, and failure recovery; +* public contracts between independently changing components; +* code that changes often or is difficult to diagnose through broader evidence. Data-transforming functions are good candidates because their inputs and outputs often provide a clear oracle. They are not automatically worth a separate test. @@ -203,22 +203,22 @@ A trivial transformation already exercised through stable public behavior may ga Ask whether a focused test would clarify a contract, distinguish meaningful cases, improve diagnosis, or enable safer change. If none applies, broader evidence may be enough. -## What makes a retained test useful? +== What makes a retained test useful? A useful behavior test normally has these properties: -- **Relevant:** it protects behavior or risk worth maintaining. -- **Discriminating:** it can fail for an important incorrect implementation. -- **Readable:** its setup, action, and expected outcome communicate the rule. -- **Independent:** its oracle does not merely reproduce production logic. -- **Stable:** structural refactoring does not break it without a behavior change. -- **Diagnostic:** a failure points to a reasonably small question. -- **Repeatable:** unchanged code and environment produce the same result. -- **Economical:** its evidence justifies its runtime and maintenance cost. +* *Relevant:* it protects behavior or risk worth maintaining. +* *Discriminating:* it can fail for an important incorrect implementation. +* *Readable:* its setup, action, and expected outcome communicate the rule. +* *Independent:* its oracle does not merely reproduce production logic. +* *Stable:* structural refactoring does not break it without a behavior change. +* *Diagnostic:* a failure points to a reasonably small question. +* *Repeatable:* unchanged code and environment produce the same result. +* *Economical:* its evidence justifies its runtime and maintenance cost. No single property proves quality. A fast, readable test of an irrelevant detail is still low value. -## Depend on behavior, not replaceable structure +== Depend on behavior, not replaceable structure A durable test depends on behavior at its chosen boundary. It should fail when that behavior changes and remain Green when only replaceable internals change. @@ -230,7 +230,7 @@ That focused test proves only that the request was made. It does not prove that The chapters on choosing evidence and collaborative discovery explain how other evidence answers those questions without forcing one test to prove everything. -## Use feedback lanes without weakening Red +== Use feedback lanes without weakening Red Tests created during TDD do not remain a separate species. Once retained, they become part of the project’s verification portfolio. @@ -240,9 +240,9 @@ If only a slower test through a real dependency or an assembled path can faithfu Do not replace faithful evidence with an irrelevant focused test merely to make the loop faster. Instead reduce setup, narrow the scenario, improve diagnostics, or create a safe controllable boundary. -The chapter on [choosing evidence and feedback lanes](04-choosing-evidence-and-feedback-lanes.md) develops fast, standard, and thorough execution lanes. +The chapter on xref:04-choosing-evidence-and-feedback-lanes.adoc[choosing evidence and feedback lanes] develops fast, standard, and thorough execution lanes. -## Keep results trustworthy +== Keep results trustworthy A flaky test damages the feedback loop because its failure no longer points reliably to changed behavior. Treat flakiness as a defect, not background noise. @@ -252,68 +252,70 @@ A broad example of stakeholder-visible behavior may guide several focused inner At completion, report what was run, what passed, what could not run, and what uncertainty remains. Never claim verification that was not observed. -## Explicit exception paths +== Explicit exception paths The normal Red requirement applies to new behavior and defect corrections. Other kinds of work begin from different evidence states. -### Characterizing legacy behavior +=== Characterizing legacy behavior A characterization test records what the system currently does so a risky change can preserve selected behavior. It usually begins Green and does not prove that the behavior is correct. If no test can run, a minimal behavior-preserving seam may be needed first. After characterization, establish a meaningful Red for the intended change. -### Pure refactoring +=== Pure refactoring Pure refactoring begins Green and remains Green because observable behavior is intentionally unchanged. If behavior must change, begin a new cycle. -### Exploration +=== Exploration A spike or experiment may be needed to discover feasibility, an interface, or an oracle. Keep exploratory code disposable, capture what was learned, then start deliberate development from an explicit behavior. -### No useful executable oracle +=== No useful executable oracle Some outcomes require human review, experiments, statistical evaluation, monitoring, or specialist analysis. Apply TDD to deterministic subproblems, but do not pretend an exact assertion proves what it cannot. -The [adjacent-practices appendix](appendices/adjacent-practices-and-further-reading.md) introduces these complementary forms of evidence. +The xref:appendices/adjacent-practices-and-further-reading.adoc[adjacent-practices appendix] introduces these complementary forms of evidence. -## The operating invariants +== The operating invariants For new behavior and defect corrections, preserve these invariants: -1. **Known baseline:** existing relevant results are known before the new example. -2. **Explicit decision:** one behavior, risk, boundary, and expected outcome are stated. -3. **Independent oracle:** the expectation is not copied from production reasoning. -4. **Observed Red:** the test fails reproducibly for the expected behavioral reason. -5. **Faithful evidence:** the test can detect the risk it claims to address. -6. **Green integrity:** production satisfies the evidence without weakening it. -7. **Safe refactoring:** structure changes only while behavior remains Green. -8. **Trustworthy result:** flaky or unexplained outcomes are not accepted as evidence. -9. **Closed cycle:** the current behavioral decision returns to Green before the next begins. -10. **Honest completion:** unrun checks and remaining uncertainty are reported. +[arabic] +. *Known baseline:* existing relevant results are known before the new example. +. *Explicit decision:* one behavior, risk, boundary, and expected outcome are stated. +. *Independent oracle:* the expectation is not copied from production reasoning. +. *Observed Red:* the test fails reproducibly for the expected behavioral reason. +. *Faithful evidence:* the test can detect the risk it claims to address. +. *Green integrity:* production satisfies the evidence without weakening it. +. *Safe refactoring:* structure changes only while behavior remains Green. +. *Trustworthy result:* flaky or unexplained outcomes are not accepted as evidence. +. *Closed cycle:* the current behavioral decision returns to Green before the next begins. +. *Honest completion:* unrun checks and remaining uncertainty are reported. These rules protect the meaning of the colors. They are not targets for assertion counts, method coverage, cycle duration, or use of simulated dependencies. -## A compact working loop +== A compact working loop For each new behavior or defect correction: -1. State the behavior and why it matters. -2. Make or revise a provisional test list. -3. Establish the relevant baseline. -4. Choose the narrowest boundary faithful to the risk. -5. Derive the expected result independently. -6. Add or revise one example and observe meaningful Red. -7. Make the smallest coherent production change. -8. Observe Green and run nearby relevant tests. -9. Refactor without changing observable behavior. -10. Close the cycle, then choose the next informative example. +[arabic] +. State the behavior and why it matters. +. Make or revise a provisional test list. +. Establish the relevant baseline. +. Choose the narrowest boundary faithful to the risk. +. Derive the expected result independently. +. Add or revise one example and observe meaningful Red. +. Make the smallest coherent production change. +. Observe Green and run nearby relevant tests. +. Refactor without changing observable behavior. +. Close the cycle, then choose the next informative example. The next chapter turns recurring misunderstandings into concrete symptoms and recovery actions. -## Sources and further reading +== Sources and further reading -- Kent Beck, [Canon TDD](https://newsletter.kentbeck.com/p/canon-tdd) -- Kent Beck, [TDD is Kanban for Code](https://newsletter.kentbeck.com/p/tdd-is-kanban-for-code) -- Martin Fowler, [Is Changing Interfaces Refactoring?](https://martinfowler.com/bliki/IsChangingInterfacesRefactoring.html) -- Michael Feathers, *Working Effectively with Legacy Code* -- Gerard Meszaros, [xUnit Test Patterns](http://xunitpatterns.com/) +* Kent Beck, https://newsletter.kentbeck.com/p/canon-tdd[Canon TDD] +* Kent Beck, https://newsletter.kentbeck.com/p/tdd-is-kanban-for-code[TDD is Kanban for Code] +* Martin Fowler, https://martinfowler.com/bliki/IsChangingInterfacesRefactoring.html[Is Changing Interfaces Refactoring?] +* Michael Feathers, _Working Effectively with Legacy Code_ +* Gerard Meszaros, http://xunitpatterns.com/[xUnit Test Patterns] diff --git a/setup/optional-skills/knowledge/tdd/guide/03-common-misconceptions-and-failure-modes.adoc b/setup/optional-skills/knowledge/tdd/guide/03-common-misconceptions-and-failure-modes.adoc new file mode 100644 index 0000000..78f03e5 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/guide/03-common-misconceptions-and-failure-modes.adoc @@ -0,0 +1,354 @@ += Common misconceptions and failure modes + +TDD can disappoint because it was misapplied, because its evidence did not match the risk, or because it was a poor fit for the work. + +Those explanations are not interchangeable. Calling every criticism a misunderstanding makes TDD unfalsifiable. Calling every failed adoption proof against TDD ignores how the practice was used. + +This chapter first diagnoses common misuse. It then separates those failure modes from serious objections that remain even when TDD is practised competently. + +Each diagnostic uses the same questions: + +* What is the misconception? +* What symptom appears in the work? +* Why does it weaken feedback or design? +* How can the team recover? +* Where is the deeper explanation? + +== Mistaking a by-product for the goal + +=== “TDD means testing every method” + +*Misconception:* Each function, method, or class must receive its own test. + +*Symptom:* The suite mirrors the code inventory. Trivial accessors receive tests while important cross-component behavior remains uncertain. + +*Why it fails:* Program structure does not determine testing value. Structure-coupled tests add maintenance cost and break during harmless refactoring. + +*Recovery:* Start from behavior and risk. Add focused evidence when it clarifies a contract, distinguishes important cases, improves diagnosis, or enables safe change. + +*Read next:* “Start from behavior, not code inventory” and “What deserves a focused test” in xref:02-practising-tdd.adoc[Practising TDD deliberately]. + +=== “Coverage is the objective” + +*Misconception:* A high coverage percentage demonstrates test quality or successful TDD. + +*Symptom:* Tests execute lines without discriminating assertions. Teams avoid valuable refactoring or write low-value tests to protect a target. + +*Why it fails:* Coverage measures execution, not correctness of the oracle, fault detection, or importance of the behavior. + +*Recovery:* Use coverage to find unexamined code, then ask which risk matters and what evidence could detect a meaningful error. Do not treat the percentage as the product. + +*Read next:* “Why coverage is not the goal” in xref:01-why-tdd.adoc[Why test-driven development exists]. + +=== “Test-first automatically produces good tests” + +*Misconception:* Writing an assertion before production code makes the resulting test valuable. + +*Symptom:* A test is fast and test-first but checks an irrelevant detail, copies production reasoning, or cannot detect a plausible defect. + +*Why it fails:* Timing creates an opportunity for design feedback. Relevance, independence, sensitivity, stability, and fidelity determine the quality of the evidence. + +*Recovery:* Review the behavior, risk, boundary, and oracle. Confirm that an important wrong implementation would fail. + +*Read next:* “Derive the expected result independently” and “What makes a retained test useful” in xref:02-practising-tdd.adoc[Practising TDD deliberately]. + +== Breaking the meaning of Red + +=== “The change is obvious, so Red can be skipped” + +*Misconception:* A developer or AI can reason that the test would have failed without executing it. + +*Symptom:* Test and implementation appear together, followed only by a passing run. Nobody knows whether the test was ever sensitive to the missing behavior. + +*Why it fails:* The test may use the wrong setup, miss the production path, contain a tautology, or assert a result the system already produced. + +*Recovery:* Run the new or revised test against the unchanged behavior. Require a reproducible failure caused by the intended rule before implementation. + +*Read next:* “Observe a meaningful Red” in xref:02-practising-tdd.adoc[Practising TDD deliberately]. + +=== “Any failure counts as Red” + +*Misconception:* A compilation error, broken fixture, missing import, or unavailable service proves the behavioral example. + +*Symptom:* Production code changes while the test has not yet reached the behavior it claims to describe. + +*Why it fails:* The failure proves only that the execution path is broken. It does not show that the oracle distinguishes the current behavior from the intended behavior. + +*Recovery:* Repair the test route with the smallest necessary structure. Restore the known baseline, then rerun until the expected behavioral failure is visible. + +*Read next:* “If the test fails for the wrong reason” in xref:02-practising-tdd.adoc[Practising TDD deliberately]. + +=== “An existing suite failure can serve as the new Red” + +*Misconception:* Any Red result in the repository is enough to begin the production change. + +*Symptom:* A pre-existing or unrelated failure is later reported as proof that the new test detected the missing behavior. + +*Why it fails:* The change has no attributable baseline. Green may hide the old failure or leave the new behavior untested. + +*Recovery:* Record or repair existing failures first. Isolate the new example and observe its own relevant failure. + +*Read next:* “Establish a known baseline” in xref:02-practising-tdd.adoc[Practising TDD deliberately]. + +=== “A new test that passes immediately is close enough” + +*Misconception:* An immediately Green test can be accepted as evidence of a completed Red–Green cycle. + +*Symptom:* The team adds production code anyway or retains an assertion without checking whether it can detect a defect. + +*Why it fails:* The behavior may already exist, be satisfied accidentally, or never be exercised by the test. + +*Recovery:* Inspect setup, action, and oracle. When safe, perturb the relevant behavior to check sensitivity. If the behavior already exists, choose a genuinely missing behavior. + +*Read next:* “If the new test passes immediately” in xref:02-practising-tdd.adoc[Practising TDD deliberately]. + +== Turning the cycle into ceremony + +=== “Write the entire test suite first” + +*Misconception:* Test-first means specifying every case before any implementation. + +*Symptom:* Many tests fail at once and encode an interface imagined before the first design feedback arrives. + +*Why it fails:* Later tests cannot benefit from discoveries made while satisfying earlier examples. Failures also become harder to attribute. + +*Recovery:* Keep a provisional test list, select one informative example, and close its cycle before choosing the next. + +*Read next:* “Make a provisional test list” in xref:02-practising-tdd.adoc[Practising TDD deliberately]. + +=== “One behavior means one assertion” + +*Misconception:* Every test may contain only one assertion. + +*Symptom:* One coherent outcome is split across tests with duplicated setup, or related facts become difficult to read together. + +*Why it fails:* Assertion count is a syntactic metric. The useful boundary is one behavioral decision and one understandable reason for failure. + +*Recovery:* Keep assertions together when they describe one outcome. Split a test when it mixes independent rules or failures would be ambiguous. + +*Read next:* “Start from behavior, not code inventory” in xref:02-practising-tdd.adoc[Practising TDD deliberately]. + +=== “Minimal Green requires deliberately bad code” + +*Misconception:* Every example must be satisfied with a hard-coded or knowingly poor implementation. + +*Symptom:* Obvious solutions are delayed by ritual, or code violates already-known requirements merely to demonstrate Fake It. + +*Why it fails:* The goal is a small, attributable learning step, not artificial incompetence. + +*Recovery:* Use Obvious Implementation when the rule is clear. Use Fake It or triangulation when a narrow implementation will expose uncertainty or useful design pressure. + +*Read next:* “Make the smallest coherent Green” in xref:02-practising-tdd.adoc[Practising TDD deliberately]. + +=== “Refactor may add behavior” + +*Misconception:* Any cleanup performed after Green belongs to Refactor, even when observable behavior changes. + +*Symptom:* New rules appear without an observed Red, and a failing test during cleanup is dismissed as an expected side effect. + +*Why it fails:* The evidence no longer distinguishes structural change from behavioral change. + +*Recovery:* Refactor only while observable behavior remains Green. Begin a new cycle when the intended contract changes. + +*Read next:* “Refactor only under Green” and “Change an existing contract deliberately” in xref:02-practising-tdd.adoc[Practising TDD deliberately]. + +== Choosing evidence by fashion + +Test *scope* describes how much of the system participates. A focused test observes a narrow coherent behavior. An integration test includes real collaborators or infrastructure. + +An end-to-end test observes an assembled path from an outside entry point. The next chapter defines these and other evidence labels in more detail. + +=== “TDD means unit tests” + +*Misconception:* Red–Green–Refactor must always operate through an isolated unit test. + +*Symptom:* A fast test bypasses the database, framework, contract, or assembly behavior that creates the current risk. + +*Why it fails:* TDD determines when feedback enters development. It does not require one fixed test scope. + +*Recovery:* Use the narrowest boundary that faithfully answers the current question. A focused, integration, or end-to-end test can be the correct Red. + +*Read next:* xref:04-choosing-evidence-and-feedback-lanes.adoc[Choosing evidence and feedback lanes]. + +=== “The narrowest test is always best” + +*Misconception:* A focused test is preferable even when it bypasses the component that creates the risk. + +*Symptom:* A suite is fast but cannot detect mapping, persistence, framework, configuration, or deployment defects. + +*Why it fails:* Raw speed is not useful when the evidence is unfaithful to the question. + +*Recovery:* Prefer the shortest trustworthy loop that faithfully exercises the current risk. Use a broader Red when the broader boundary is what can fail. + +*Read next:* xref:04-choosing-evidence-and-feedback-lanes.adoc[Choosing evidence and feedback lanes]. + +=== “End-to-end tests are always better because they are realistic” + +*Misconception:* The broadest available boundary provides the strongest evidence for every question. + +*Symptom:* Inner development cycles become slow and failures are hard to localize. Many scenarios repeat the same expensive setup. + +*Why it fails:* Broad tests include more collaborating parts, but they may still use unrealistic data or simulated services. Scope, fidelity, speed, and diagnosis are separate qualities. + +*Recovery:* Use focused evidence for local rules and broader evidence for assembly, journeys, and deployment risks. Keep only scenarios whose additional scope answers a distinct question. + +*Read next:* xref:04-choosing-evidence-and-feedback-lanes.adoc[Choosing evidence and feedback lanes]. + +=== “One test level proves the whole feature” + +*Misconception:* A focused, integration, or end-to-end test can answer every relevant question about one behavior. + +*Symptom:* A team treats a recorded request as proof of an external effect, or repeats every local rule through the browser. + +*Why it fails:* Different boundaries expose different failures. One focused interaction does not prove provider acceptance, and one broad journey may diagnose a local rule poorly. + +*Recovery:* Map each material risk to the smallest faithful evidence. Remove tests that duplicate another test’s question without adding useful confidence. + +*Read next:* xref:04-choosing-evidence-and-feedback-lanes.adoc[Choosing evidence and feedback lanes] and the xref:07-worked-examples.adoc[worked examples]. + +== Confusing testability with mockability + +A *test double* is a controlled substitute for a collaborator. A *mock* is a double used to set or verify expected interactions. + +=== “Every collaborator should be mocked” + +*Misconception:* TDD requires replacing every dependency with a test double. + +*Symptom:* Interfaces exist only for tests, assertions mirror internal call order, and refactoring breaks many tests without changing behavior. + +*Why it fails:* Doubles can improve control and diagnosis, but excessive isolation couples tests to imagined implementation structure. + +*Recovery:* First look for a stable behavioral boundary. Use a double when control or observation is difficult and the asserted interaction is a meaningful contract. + +*Read next:* “Testability rather than mockability” in the xref:07-worked-examples.adoc[worked examples]. + +=== “A mock proves the external effect” + +*Misconception:* Verifying that code requested a charge, message, or write proves that the real system completed it. + +*Symptom:* Focused tests pass while provider contracts, credentials, serialization, settlement, or reconciliation fail. + +*Why it fails:* The test proves only the interaction visible at its boundary. + +*Recovery:* Keep the focused interaction test when the request matters. Add compatibility, integration, end-to-end, or operational evidence for the external effect and its failure modes. + +*Read next:* xref:04-choosing-evidence-and-feedback-lanes.adoc[Choosing evidence and feedback lanes]. + +== Misusing retained tests + +=== “Tests must never change” + +*Misconception:* Changing an existing test is equivalent to weakening it. + +*Symptom:* The suite preserves obsolete requirements, or developers work around a test instead of updating an intentional contract change. + +*Why it fails:* Retained tests are executable specifications. A specification must change when the intended behavior changes. + +*Recovery:* Distinguish an intentional contract change from an accidental regression. Revise the expectation first, observe Red, update production behavior, and preserve unaffected evidence. + +*Read next:* “Change an existing contract deliberately” in xref:02-practising-tdd.adoc[Practising TDD deliberately]. + +=== “A characterization test proves legacy behavior is correct” + +*Misconception:* Recording current output means the output is desired. + +*Symptom:* Defects and historical accidents become protected as requirements without review. + +*Why it fails:* A characterization test establishes what happens now, not what should happen. + +*Recovery:* Name the behavior being preserved and why. Review suspicious output with a domain owner, then establish Red for any intentional correction. + +*Read next:* The legacy sequence in xref:06-applying-tdd-without-dogma.adoc[Applying TDD without dogma] and the xref:07-worked-examples.adoc[worked examples]. + +=== “A flaky test can be rerun until Green” + +*Misconception:* A passing rerun is adequate verification. + +*Symptom:* Failures are ignored when a later execution happens to pass. The suite’s colors no longer carry a stable meaning. + +*Why it fails:* An unexplained result cannot be attributed to code behavior. It may hide shared state, races, time, infrastructure faults, or a weak oracle. + +*Recovery:* Treat flakiness as a defect. Reproduce and remove its cause, or exclude the test from required gates while clearly reporting the lost evidence. + +*Read next:* “Keep results trustworthy” in xref:02-practising-tdd.adoc[Practising TDD deliberately]. + +== Asking TDD to solve a different problem + +=== “TDD replaces requirement discovery” + +*Misconception:* Writing examples eliminates the need to understand users, goals, constraints, and competing interpretations. + +*Symptom:* Precise tests encode the wrong behavior, and passing software still fails the user’s need. + +*Why it fails:* TDD checks an expressed expectation. It cannot decide whether that expectation is valuable or complete. + +*Recovery:* Use collaborative examples, product discovery, domain expertise, and review to choose behavior. Use TDD to develop the agreed behavior in small, observable steps. + +*Read next:* xref:05-bdd-and-collaborative-discovery.adoc[Behavior-driven development (BDD) and collaborative discovery]. + +=== “TDD grows the whole architecture automatically” + +*Misconception:* Local test-first decisions remove the need for architectural reasoning, experiments, integration, and operational feedback. + +*Symptom:* Local units are easy to test, but the assembled system has poor boundaries, delivery risks, or quality-attribute failures. + +*Why it fails:* A TDD cycle provides local design feedback at its observation boundary. It does not by itself compare system-level options or validate operation at scale. + +*Recovery:* Combine small cycles with a thin runnable system slice, disposable experiments, deliberate architecture, broader evidence, and production feedback. + +*Read next:* xref:06-applying-tdd-without-dogma.adoc[Applying TDD without dogma] and the xref:appendices/adjacent-practices-and-further-reading.adoc[adjacent-practices appendix]. + +=== “Every valuable outcome has an exact oracle” + +*Misconception:* Every quality can be reduced to one deterministic expected value. + +*Symptom:* Subjective, statistical, emergent, or operational qualities receive misleading assertions because the workflow expects Red–Green. + +*Why it fails:* Some questions require ranges, properties, expert review, experiments, monitoring, or repeated evaluation. + +*Recovery:* Apply exact example-based TDD to deterministic subproblems. Use the form of evidence appropriate to the remaining uncertainty. + +*Read next:* “No useful executable oracle” in xref:02-practising-tdd.adoc[Practising TDD deliberately] and the xref:appendices/adjacent-practices-and-further-reading.adoc[adjacent-practices appendix]. + +== Serious objections are not misconceptions + +The following concerns can remain valid even when no failure mode above is present. They should be evaluated against a concrete alternative, not dismissed as resistance to discipline. + +They include: + +* the cycle and retained suite may cost more than the feedback repays; +* isolation-heavy tests can damage production design; +* strict micro-cycles may fit schema, distributed, or cross-cutting changes poorly; +* empirical results vary by task, team, experience, and comparison process; +* local design feedback cannot choose or validate the whole system architecture. + +These are evaluation questions, not errors to “recover” from. xref:06-applying-tdd-without-dogma.adoc[Applying TDD without dogma] examines their evidence, trade-offs, and adaptations. + +== A recovery sequence + +When a TDD effort feels slow or unhelpful: + +[arabic] +. State the behavior and current risk again. +. Check whether the oracle is independent and meaningful. +. Confirm that Red was observed for the expected reason. +. Check whether the boundary faithfully exercises the risk. +. Remove structural coupling and unnecessary setup. +. Restore trustworthy Green before starting another behavior. +. Choose different evidence when TDD is not the right mechanism. + +The goal is not ritual compliance. It is early, trustworthy information that improves the next development decision. + +== Sources + +The historical and empirical basis for this chapter is summarized in +xref:../research-findings.adoc[the research findings]. Useful primary or +canonical references include: + +* https://martinfowler.com/bliki/TestDrivenDevelopment.html[Martin Fowler on + Test-Driven Development] +* https://testing.googleblog.com/2020/08/code-coverage-best-practices.html[Google + Testing Blog on coverage] +* https://martinfowler.com/articles/mocksArentStubs.html[Martin Fowler on mocks + and stubs] +* https://cucumber.io/docs/bdd/[Cucumber documentation on BDD] diff --git a/setup/optional-skills/knowledge/tdd/guide/03-common-misconceptions-and-failure-modes.md b/setup/optional-skills/knowledge/tdd/guide/03-common-misconceptions-and-failure-modes.md deleted file mode 100644 index a7dfdcb..0000000 --- a/setup/optional-skills/knowledge/tdd/guide/03-common-misconceptions-and-failure-modes.md +++ /dev/null @@ -1,339 +0,0 @@ -# Common misconceptions and failure modes - -TDD can disappoint because it was misapplied, because its evidence did not match the risk, or because it was a poor fit for the work. - -Those explanations are not interchangeable. Calling every criticism a misunderstanding makes TDD unfalsifiable. Calling every failed adoption proof against TDD ignores how the practice was used. - -This chapter first diagnoses common misuse. It then separates those failure modes from serious objections that remain even when TDD is practised competently. - -Each diagnostic uses the same questions: - -- What is the misconception? -- What symptom appears in the work? -- Why does it weaken feedback or design? -- How can the team recover? -- Where is the deeper explanation? - -## Mistaking a by-product for the goal - -### “TDD means testing every method” - -**Misconception:** Each function, method, or class must receive its own test. - -**Symptom:** The suite mirrors the code inventory. Trivial accessors receive tests while important cross-component behavior remains uncertain. - -**Why it fails:** Program structure does not determine testing value. Structure-coupled tests add maintenance cost and break during harmless refactoring. - -**Recovery:** Start from behavior and risk. Add focused evidence when it clarifies a contract, distinguishes important cases, improves diagnosis, or enables safe change. - -**Read next:** “Start from behavior, not code inventory” and “What deserves a focused test” in [Practising TDD deliberately](02-practising-tdd.md). - -### “Coverage is the objective” - -**Misconception:** A high coverage percentage demonstrates test quality or successful TDD. - -**Symptom:** Tests execute lines without discriminating assertions. Teams avoid valuable refactoring or write low-value tests to protect a target. - -**Why it fails:** Coverage measures execution, not correctness of the oracle, fault detection, or importance of the behavior. - -**Recovery:** Use coverage to find unexamined code, then ask which risk matters and what evidence could detect a meaningful error. Do not treat the percentage as the product. - -**Read next:** “Why coverage is not the goal” in [Why test-driven development exists](01-why-tdd.md). - -### “Test-first automatically produces good tests” - -**Misconception:** Writing an assertion before production code makes the resulting test valuable. - -**Symptom:** A test is fast and test-first but checks an irrelevant detail, copies production reasoning, or cannot detect a plausible defect. - -**Why it fails:** Timing creates an opportunity for design feedback. Relevance, independence, sensitivity, stability, and fidelity determine the quality of the evidence. - -**Recovery:** Review the behavior, risk, boundary, and oracle. Confirm that an important wrong implementation would fail. - -**Read next:** “Derive the expected result independently” and “What makes a retained test useful” in [Practising TDD deliberately](02-practising-tdd.md). - -## Breaking the meaning of Red - -### “The change is obvious, so Red can be skipped” - -**Misconception:** A developer or AI can reason that the test would have failed without executing it. - -**Symptom:** Test and implementation appear together, followed only by a passing run. Nobody knows whether the test was ever sensitive to the missing behavior. - -**Why it fails:** The test may use the wrong setup, miss the production path, contain a tautology, or assert a result the system already produced. - -**Recovery:** Run the new or revised test against the unchanged behavior. Require a reproducible failure caused by the intended rule before implementation. - -**Read next:** “Observe a meaningful Red” in [Practising TDD deliberately](02-practising-tdd.md). - -### “Any failure counts as Red” - -**Misconception:** A compilation error, broken fixture, missing import, or unavailable service proves the behavioral example. - -**Symptom:** Production code changes while the test has not yet reached the behavior it claims to describe. - -**Why it fails:** The failure proves only that the execution path is broken. It does not show that the oracle distinguishes the current behavior from the intended behavior. - -**Recovery:** Repair the test route with the smallest necessary structure. Restore the known baseline, then rerun until the expected behavioral failure is visible. - -**Read next:** “If the test fails for the wrong reason” in [Practising TDD deliberately](02-practising-tdd.md). - -### “An existing suite failure can serve as the new Red” - -**Misconception:** Any Red result in the repository is enough to begin the production change. - -**Symptom:** A pre-existing or unrelated failure is later reported as proof that the new test detected the missing behavior. - -**Why it fails:** The change has no attributable baseline. Green may hide the old failure or leave the new behavior untested. - -**Recovery:** Record or repair existing failures first. Isolate the new example and observe its own relevant failure. - -**Read next:** “Establish a known baseline” in [Practising TDD deliberately](02-practising-tdd.md). - -### “A new test that passes immediately is close enough” - -**Misconception:** An immediately Green test can be accepted as evidence of a completed Red–Green cycle. - -**Symptom:** The team adds production code anyway or retains an assertion without checking whether it can detect a defect. - -**Why it fails:** The behavior may already exist, be satisfied accidentally, or never be exercised by the test. - -**Recovery:** Inspect setup, action, and oracle. When safe, perturb the relevant behavior to check sensitivity. If the behavior already exists, choose a genuinely missing behavior. - -**Read next:** “If the new test passes immediately” in [Practising TDD deliberately](02-practising-tdd.md). - -## Turning the cycle into ceremony - -### “Write the entire test suite first” - -**Misconception:** Test-first means specifying every case before any implementation. - -**Symptom:** Many tests fail at once and encode an interface imagined before the first design feedback arrives. - -**Why it fails:** Later tests cannot benefit from discoveries made while satisfying earlier examples. Failures also become harder to attribute. - -**Recovery:** Keep a provisional test list, select one informative example, and close its cycle before choosing the next. - -**Read next:** “Make a provisional test list” in [Practising TDD deliberately](02-practising-tdd.md). - -### “One behavior means one assertion” - -**Misconception:** Every test may contain only one assertion. - -**Symptom:** One coherent outcome is split across tests with duplicated setup, or related facts become difficult to read together. - -**Why it fails:** Assertion count is a syntactic metric. The useful boundary is one behavioral decision and one understandable reason for failure. - -**Recovery:** Keep assertions together when they describe one outcome. Split a test when it mixes independent rules or failures would be ambiguous. - -**Read next:** “Start from behavior, not code inventory” in [Practising TDD deliberately](02-practising-tdd.md). - -### “Minimal Green requires deliberately bad code” - -**Misconception:** Every example must be satisfied with a hard-coded or knowingly poor implementation. - -**Symptom:** Obvious solutions are delayed by ritual, or code violates already-known requirements merely to demonstrate Fake It. - -**Why it fails:** The goal is a small, attributable learning step, not artificial incompetence. - -**Recovery:** Use Obvious Implementation when the rule is clear. Use Fake It or triangulation when a narrow implementation will expose uncertainty or useful design pressure. - -**Read next:** “Make the smallest coherent Green” in [Practising TDD deliberately](02-practising-tdd.md). - -### “Refactor may add behavior” - -**Misconception:** Any cleanup performed after Green belongs to Refactor, even when observable behavior changes. - -**Symptom:** New rules appear without an observed Red, and a failing test during cleanup is dismissed as an expected side effect. - -**Why it fails:** The evidence no longer distinguishes structural change from behavioral change. - -**Recovery:** Refactor only while observable behavior remains Green. Begin a new cycle when the intended contract changes. - -**Read next:** “Refactor only under Green” and “Change an existing contract deliberately” in [Practising TDD deliberately](02-practising-tdd.md). - -## Choosing evidence by fashion - -Test **scope** describes how much of the system participates. A focused test observes a narrow coherent behavior. An integration test includes real collaborators or infrastructure. - -An end-to-end test observes an assembled path from an outside entry point. The next chapter defines these and other evidence labels in more detail. - -### “TDD means unit tests” - -**Misconception:** Red–Green–Refactor must always operate through an isolated unit test. - -**Symptom:** A fast test bypasses the database, framework, contract, or assembly behavior that creates the current risk. - -**Why it fails:** TDD determines when feedback enters development. It does not require one fixed test scope. - -**Recovery:** Use the narrowest boundary that faithfully answers the current question. A focused, integration, or end-to-end test can be the correct Red. - -**Read next:** [Choosing evidence and feedback lanes](04-choosing-evidence-and-feedback-lanes.md). - -### “The narrowest test is always best” - -**Misconception:** A focused test is preferable even when it bypasses the component that creates the risk. - -**Symptom:** A suite is fast but cannot detect mapping, persistence, framework, configuration, or deployment defects. - -**Why it fails:** Raw speed is not useful when the evidence is unfaithful to the question. - -**Recovery:** Prefer the shortest trustworthy loop that faithfully exercises the current risk. Use a broader Red when the broader boundary is what can fail. - -**Read next:** [Choosing evidence and feedback lanes](04-choosing-evidence-and-feedback-lanes.md). - -### “End-to-end tests are always better because they are realistic” - -**Misconception:** The broadest available boundary provides the strongest evidence for every question. - -**Symptom:** Inner development cycles become slow and failures are hard to localize. Many scenarios repeat the same expensive setup. - -**Why it fails:** Broad tests include more collaborating parts, but they may still use unrealistic data or simulated services. Scope, fidelity, speed, and diagnosis are separate qualities. - -**Recovery:** Use focused evidence for local rules and broader evidence for assembly, journeys, and deployment risks. Keep only scenarios whose additional scope answers a distinct question. - -**Read next:** [Choosing evidence and feedback lanes](04-choosing-evidence-and-feedback-lanes.md). - -### “One test level proves the whole feature” - -**Misconception:** A focused, integration, or end-to-end test can answer every relevant question about one behavior. - -**Symptom:** A team treats a recorded request as proof of an external effect, or repeats every local rule through the browser. - -**Why it fails:** Different boundaries expose different failures. One focused interaction does not prove provider acceptance, and one broad journey may diagnose a local rule poorly. - -**Recovery:** Map each material risk to the smallest faithful evidence. Remove tests that duplicate another test’s question without adding useful confidence. - -**Read next:** [Choosing evidence and feedback lanes](04-choosing-evidence-and-feedback-lanes.md) and the [worked examples](07-worked-examples.md). - -## Confusing testability with mockability - -A **test double** is a controlled substitute for a collaborator. A **mock** is a double used to set or verify expected interactions. - -### “Every collaborator should be mocked” - -**Misconception:** TDD requires replacing every dependency with a test double. - -**Symptom:** Interfaces exist only for tests, assertions mirror internal call order, and refactoring breaks many tests without changing behavior. - -**Why it fails:** Doubles can improve control and diagnosis, but excessive isolation couples tests to imagined implementation structure. - -**Recovery:** First look for a stable behavioral boundary. Use a double when control or observation is difficult and the asserted interaction is a meaningful contract. - -**Read next:** “Testability rather than mockability” in the [worked examples](07-worked-examples.md). - -### “A mock proves the external effect” - -**Misconception:** Verifying that code requested a charge, message, or write proves that the real system completed it. - -**Symptom:** Focused tests pass while provider contracts, credentials, serialization, settlement, or reconciliation fail. - -**Why it fails:** The test proves only the interaction visible at its boundary. - -**Recovery:** Keep the focused interaction test when the request matters. Add compatibility, integration, end-to-end, or operational evidence for the external effect and its failure modes. - -**Read next:** [Choosing evidence and feedback lanes](04-choosing-evidence-and-feedback-lanes.md). - -## Misusing retained tests - -### “Tests must never change” - -**Misconception:** Changing an existing test is equivalent to weakening it. - -**Symptom:** The suite preserves obsolete requirements, or developers work around a test instead of updating an intentional contract change. - -**Why it fails:** Retained tests are executable specifications. A specification must change when the intended behavior changes. - -**Recovery:** Distinguish an intentional contract change from an accidental regression. Revise the expectation first, observe Red, update production behavior, and preserve unaffected evidence. - -**Read next:** “Change an existing contract deliberately” in [Practising TDD deliberately](02-practising-tdd.md). - -### “A characterization test proves legacy behavior is correct” - -**Misconception:** Recording current output means the output is desired. - -**Symptom:** Defects and historical accidents become protected as requirements without review. - -**Why it fails:** A characterization test establishes what happens now, not what should happen. - -**Recovery:** Name the behavior being preserved and why. Review suspicious output with a domain owner, then establish Red for any intentional correction. - -**Read next:** The legacy sequence in [Applying TDD without dogma](06-applying-tdd-without-dogma.md) and the [worked examples](07-worked-examples.md). - -### “A flaky test can be rerun until Green” - -**Misconception:** A passing rerun is adequate verification. - -**Symptom:** Failures are ignored when a later execution happens to pass. The suite’s colors no longer carry a stable meaning. - -**Why it fails:** An unexplained result cannot be attributed to code behavior. It may hide shared state, races, time, infrastructure faults, or a weak oracle. - -**Recovery:** Treat flakiness as a defect. Reproduce and remove its cause, or exclude the test from required gates while clearly reporting the lost evidence. - -**Read next:** “Keep results trustworthy” in [Practising TDD deliberately](02-practising-tdd.md). - -## Asking TDD to solve a different problem - -### “TDD replaces requirement discovery” - -**Misconception:** Writing examples eliminates the need to understand users, goals, constraints, and competing interpretations. - -**Symptom:** Precise tests encode the wrong behavior, and passing software still fails the user’s need. - -**Why it fails:** TDD checks an expressed expectation. It cannot decide whether that expectation is valuable or complete. - -**Recovery:** Use collaborative examples, product discovery, domain expertise, and review to choose behavior. Use TDD to develop the agreed behavior in small, observable steps. - -**Read next:** [Behavior-driven development (BDD) and collaborative discovery](05-bdd-and-collaborative-discovery.md). - -### “TDD grows the whole architecture automatically” - -**Misconception:** Local test-first decisions remove the need for architectural reasoning, experiments, integration, and operational feedback. - -**Symptom:** Local units are easy to test, but the assembled system has poor boundaries, delivery risks, or quality-attribute failures. - -**Why it fails:** A TDD cycle provides local design feedback at its observation boundary. It does not by itself compare system-level options or validate operation at scale. - -**Recovery:** Combine small cycles with a thin runnable system slice, disposable experiments, deliberate architecture, broader evidence, and production feedback. - -**Read next:** [Applying TDD without dogma](06-applying-tdd-without-dogma.md) and the [adjacent-practices appendix](appendices/adjacent-practices-and-further-reading.md). - -### “Every valuable outcome has an exact oracle” - -**Misconception:** Every quality can be reduced to one deterministic expected value. - -**Symptom:** Subjective, statistical, emergent, or operational qualities receive misleading assertions because the workflow expects Red–Green. - -**Why it fails:** Some questions require ranges, properties, expert review, experiments, monitoring, or repeated evaluation. - -**Recovery:** Apply exact example-based TDD to deterministic subproblems. Use the form of evidence appropriate to the remaining uncertainty. - -**Read next:** “No useful executable oracle” in [Practising TDD deliberately](02-practising-tdd.md) and the [adjacent-practices appendix](appendices/adjacent-practices-and-further-reading.md). - -## Serious objections are not misconceptions - -The following concerns can remain valid even when no failure mode above is present. They should be evaluated against a concrete alternative, not dismissed as resistance to discipline. - -They include: - -- the cycle and retained suite may cost more than the feedback repays; -- isolation-heavy tests can damage production design; -- strict micro-cycles may fit schema, distributed, or cross-cutting changes poorly; -- empirical results vary by task, team, experience, and comparison process; -- local design feedback cannot choose or validate the whole system architecture. - -These are evaluation questions, not errors to “recover” from. [Applying TDD without dogma](06-applying-tdd-without-dogma.md) examines their evidence, trade-offs, and adaptations. - -## A recovery sequence - -When a TDD effort feels slow or unhelpful: - -1. State the behavior and current risk again. -2. Check whether the oracle is independent and meaningful. -3. Confirm that Red was observed for the expected reason. -4. Check whether the boundary faithfully exercises the risk. -5. Remove structural coupling and unnecessary setup. -6. Restore trustworthy Green before starting another behavior. -7. Choose different evidence when TDD is not the right mechanism. - -The goal is not ritual compliance. It is early, trustworthy information that improves the next development decision. diff --git a/setup/optional-skills/knowledge/tdd/guide/04-choosing-evidence-and-feedback-lanes.md b/setup/optional-skills/knowledge/tdd/guide/04-choosing-evidence-and-feedback-lanes.adoc similarity index 64% rename from setup/optional-skills/knowledge/tdd/guide/04-choosing-evidence-and-feedback-lanes.md rename to setup/optional-skills/knowledge/tdd/guide/04-choosing-evidence-and-feedback-lanes.adoc index c3426a6..9d1af74 100644 --- a/setup/optional-skills/knowledge/tdd/guide/04-choosing-evidence-and-feedback-lanes.md +++ b/setup/optional-skills/knowledge/tdd/guide/04-choosing-evidence-and-feedback-lanes.adoc @@ -1,4 +1,4 @@ -# Choosing evidence and feedback lanes += Choosing evidence and feedback lanes The TDD loop tells you when to seek feedback. It does not tell you which parts of the system must participate in each check. @@ -6,11 +6,13 @@ A pricing rule can be driven through a focused test. A transaction rule may need The governing rule is: -> Prefer the shortest trustworthy feedback loop that faithfully exercises the current risk. +____ +Prefer the shortest trustworthy feedback loop that faithfully exercises the current risk. +____ Fast feedback matters only when it is relevant, repeatable, diagnostic, and capable of detecting the failure under discussion. -## Workflow, scope, and fidelity are separate decisions +== Workflow, scope, and fidelity are separate decisions TDD is a workflow: express unsupported behavior, observe a meaningful failure, make the behavior work, and improve the design under passing evidence. @@ -22,19 +24,21 @@ Nor does broad scope guarantee realism. A test can traverse many layers while us Choose the observation boundary from the question and risk. Do not choose it from a preferred label. -## What the common labels usually ask +== What the common labels usually ask Teams use test labels differently. Agree on local meanings, but preserve the distinct questions behind them. -| Evidence | Main question | Common limitation | -| --------------- | --------------------------------------------------------------------- | --------------------------------------------------- | -| Focused or unit | Does one rule or coherent unit of behavior hold at a narrow boundary? | May omit infrastructure and assembly semantics | -| Component | Does a substantial part work through its public boundary? | May replace external systems or deployment wiring | -| Integration | Do real components or infrastructure collaborate correctly? | Setup and diagnosis often cost more | -| Contract | Do independently changing parties agree on exercised interactions? | Does not prove full provider behavior or deployment | -| Acceptance | Does a stakeholder-visible capability satisfy agreed examples? | May not use a fully deployed production-like path | -| E2E or system | Does an assembled path work from an outside observer's view? | Usually slower and harder to diagnose | -| Smoke | Is the deployed system basically reachable and operable? | Intentionally shallow | +[cols=",,",options="header",] +|=== +|Evidence |Main question |Common limitation +|Focused or unit |Does one rule or coherent unit of behavior hold at a narrow boundary? |May omit infrastructure and assembly semantics +|Component |Does a substantial part work through its public boundary? |May replace external systems or deployment wiring +|Integration |Do real components or infrastructure collaborate correctly? |Setup and diagnosis often cost more +|Contract |Do independently changing parties agree on exercised interactions? |Does not prove full provider behavior or deployment +|Acceptance |Does a stakeholder-visible capability satisfy agreed examples? |May not use a fully deployed production-like path +|E2E or system |Does an assembled path work from an outside observer's view? |Usually slower and harder to diagnose +|Smoke |Is the deployed system basically reachable and operable? |Intentionally shallow +|=== A focused test need not mean one method or class. Its unit can be any coherent behavior that is fast to control, observe, and diagnose. @@ -44,24 +48,25 @@ A contract test may be consumer-driven, provider-owned, or schema-based. Its def An acceptance test concerns agreed capability. An E2E test concerns assembled operation. One test may satisfy both descriptions, but the questions remain useful when deciding what its result proves. -## One behavior can need distinct evidence +== One behavior can need distinct evidence Suppose checkout calculates a total, stores an order, requests payment, and shows confirmation. A useful evidence set might contain: -1. focused examples for pricing and order-state rules; -2. a database integration test for constraints and transaction behavior; -3. a contract test for the payment request and expected response; -4. an acceptance example for the stakeholder-visible outcome; -5. one E2E check for deployed checkout wiring; -6. monitoring and reconciliation for real settlement failures. +[arabic] +. focused examples for pricing and order-state rules; +. a database integration test for constraints and transaction behavior; +. a contract test for the payment request and expected response; +. an acceptance example for the stakeholder-visible outcome; +. one E2E check for deployed checkout wiring; +. monitoring and reconciliation for real settlement failures. No single item proves the whole behavior. The mock that records a charge request cannot prove settlement. The E2E path need not repeat every pricing partition already established at a cheaper boundary. Keep evidence when it answers a unique, material question. Remove duplication when a cheaper check answers the same question and the broader check covers no additional risk. -## Evidence fidelity follows the risk +== Evidence fidelity follows the risk Use real infrastructure when its semantics are part of the claim. Examples include transactions, SQL collation, serialization, filesystem behavior, queues, framework wiring, and deployed configuration. @@ -71,31 +76,31 @@ If only an integration or E2E test can demonstrate the missing behavior, that sl You may later add a faster companion test for diagnosis or design feedback. It complements the broader evidence; it does not retroactively prove the broader risk. -## Feedback lanes are execution cadences +== Feedback lanes are execution cadences Retained tests become part of the project's verification portfolio. They are not permanently a separate class called “TDD tests.” How often a check runs is another decision. A focused test can be slow, and an integration test can be fast. Assign lanes from measured cost, reliability, diagnostic value, environment needs, and risk. -### Fast or local lane +=== Fast or local lane This lane supports the immediate development loop. It normally includes the new or revised test, nearby deterministic checks, and other tests that return useful local feedback quickly. Its purpose is not to prove the entire change safe. It keeps each behavioral and structural decision small enough to diagnose. -### Standard or change-validation lane +=== Standard or change-validation lane This lane establishes confidence in the affected area before handoff or integration. It can include the changed module, relevant components, integrations, and contracts. Its composition should follow change impact. “Standard” means the team's normal evidence for a change, not every check in the repository. -### Thorough or system-validation lane +=== Thorough or system-validation lane This lane supplies broader evidence before a risky transition. It may include the full relevant suite, E2E paths, environment-dependent checks, and specialized quality evaluation. Typical triggers are merge, release, infrastructure change, high-impact migration, or a scheduled verification run. The trigger depends on project risk and delivery design. -## Do not impose universal time limits +== Do not impose universal time limits A five-second suite may feel slow in one inner loop and excellent in another system. A ten-minute suite may be acceptable before merge but unusable after every edit. @@ -103,22 +108,23 @@ Measure actual duration and feedback delay. Then give each lane a project-specif Revisit the classification as tests, infrastructure, and risks change. A lane is an operating agreement, not an intrinsic property stored forever in a test name. -## Cadence through a TDD change +== Cadence through a TDD change Use the lanes without weakening the Red–Green–Refactor invariants: -1. Before Red, run the smallest relevant baseline or record known failures. -2. For Red, run the new or revised test and inspect its expected failure. -3. For Green, run that test plus nearby fast regressions. -4. During refactoring, rerun the fast lane after each meaningful structural step. -5. Before handoff or integration, run the standard lane. -6. Before the applicable risky transition, run the thorough lane. +[arabic] +. Before Red, run the smallest relevant baseline or record known failures. +. For Red, run the new or revised test and inspect its expected failure. +. For Green, run that test plus nearby fast regressions. +. During refactoring, rerun the fast lane after each meaningful structural step. +. Before handoff or integration, run the standard lane. +. Before the applicable risky transition, run the thorough lane. If the faithful Red is slow, run it when the behavior changes and use faster supporting checks between those runs. Never claim the slow risk is verified when only its companion checks ran. Report which lanes and commands ran, their results, which broader evidence did not run, and the uncertainty that remains. -## Continuous integration extends the feedback loop +== Continuous integration extends the feedback loop Local lanes help one developer. Continuous integration tests whether the combined work still agrees at the team boundary. @@ -130,23 +136,23 @@ An unfinished outer example can remain pending, branch-local, or excluded from r Do not rerun a flaky lane until it happens to pass. Diagnose or explicitly isolate the unreliable check, record the lost evidence, and restore trust promptly. -## A practical selection checklist +== A practical selection checklist Before adding or assigning a check, ask: -- What incorrect behavior or risk must this evidence detect? -- At which boundary can that outcome be controlled and observed? -- Which real semantics must participate for the result to be faithful? -- Is a cheaper check already answering the same question? -- How quickly and reliably does this check return diagnostic feedback? -- In which lane will its result influence a useful decision? +* What incorrect behavior or risk must this evidence detect? +* At which boundary can that outcome be controlled and observed? +* Which real semantics must participate for the result to be faithful? +* Is a cheaper check already answering the same question? +* How quickly and reliably does this check return diagnostic feedback? +* In which lane will its result influence a useful decision? The next chapter addresses an earlier question: how a team discovers and expresses the behavior that these checks should evaluate. -## Sources and further reading +== Sources and further reading -- Martin Fowler, [Test-Driven Development](https://martinfowler.com/bliki/TestDrivenDevelopment.html) -- Martin Fowler, [The Practical Test Pyramid](https://martinfowler.com/articles/practical-test-pyramid.html) -- Pact, [Introduction to contract testing](https://docs.pact.io/) -- Jez Humble, [Continuous Delivery](https://continuousdelivery.com/) -- Kent Beck, [TDD is Kanban for Code](https://newsletter.kentbeck.com/p/tdd-is-kanban-for-code) +* Martin Fowler, https://martinfowler.com/bliki/TestDrivenDevelopment.html[Test-Driven Development] +* Martin Fowler, https://martinfowler.com/articles/practical-test-pyramid.html[The Practical Test Pyramid] +* Pact, https://docs.pact.io/[Introduction to contract testing] +* Jez Humble, https://continuousdelivery.com/[Continuous Delivery] +* Kent Beck, https://newsletter.kentbeck.com/p/tdd-is-kanban-for-code[TDD is Kanban for Code] diff --git a/setup/optional-skills/knowledge/tdd/guide/05-bdd-and-collaborative-discovery.md b/setup/optional-skills/knowledge/tdd/guide/05-bdd-and-collaborative-discovery.adoc similarity index 61% rename from setup/optional-skills/knowledge/tdd/guide/05-bdd-and-collaborative-discovery.md rename to setup/optional-skills/knowledge/tdd/guide/05-bdd-and-collaborative-discovery.adoc index 272eccd..cf70492 100644 --- a/setup/optional-skills/knowledge/tdd/guide/05-bdd-and-collaborative-discovery.md +++ b/setup/optional-skills/knowledge/tdd/guide/05-bdd-and-collaborative-discovery.adoc @@ -1,4 +1,4 @@ -# BDD and collaborative discovery += BDD and collaborative discovery The earlier chapters assumed that one behavior could be stated clearly enough to drive a cycle. That is a useful way to learn TDD, but real product work often begins before such clarity exists. @@ -8,7 +8,7 @@ TDD can make a developer's interpretation precise. It cannot, by itself, prove t Behavior-Driven Development and related practices improve the input to the development loop through shared language, concrete examples, and collaboration. -## Why BDD entered the conversation +== Why BDD entered the conversation Dan North developed BDD while looking for better ways to teach and practise TDD. Learners struggled with where to start, what to test, how much to test at once, and how to name tests. @@ -18,15 +18,15 @@ BDD later grew beyond a teaching reformulation. Its practice commonly includes d It is therefore misleading to call BDD either wholly separate from TDD or merely TDD with renamed tests. They overlap, while BDD places stronger emphasis on collaborative discovery and shared understanding. -## Behavior belongs to a caller +== Behavior belongs to a caller The caller of a behavior need not be a person using a browser. -- A customer may use a web application. -- Another service may use an API. -- Application code may use a library. -- An operator may use a command or alert. -- A reporting system may use a data product. +* A customer may use a web application. +* Another service may use an API. +* Application code may use a library. +* An operator may use a command or alert. +* A reporting system may use a data product. Describe the result in that caller's vocabulary and at a boundary where the caller can observe it. @@ -34,13 +34,13 @@ Describe the result in that caller's vocabulary and at a boundary where the call The caller perspective does not forbid interaction checks. Requesting a charge exactly once may be part of a component's contract. The check proves the request, not that an external provider settled it. -## Given–When–Then clarifies an example +== Given–When–Then clarifies an example -BDD examples often use **Given–When–Then**: +BDD examples often use *Given–When–Then*: -- **Given:** the relevant starting situation; -- **When:** the event or action under discussion; -- **Then:** the outcome visible at the chosen boundary. +* *Given:* the relevant starting situation; +* *When:* the event or action under discussion; +* *Then:* the outcome visible at the chosen boundary. This structure separates context, trigger, and outcome. It can expose an example that combines several behaviors or observes private structure instead of a useful result. @@ -50,20 +50,20 @@ Gherkin and tools such as Cucumber are optional. They earn their cost when scena A normal test framework is often clearer when developers are the only audience. Tool choice does not determine whether the team is practising BDD. -## Discovery precedes automation +== Discovery precedes automation Automation preserves an interpretation. Collaborative discovery tests that interpretation before code makes it expensive. Useful conversations examine: -- the business rule and its purpose; -- concrete examples and counterexamples; -- boundaries, exceptions, and failure outcomes; -- vocabulary that means the same thing to each participant; -- unanswered questions and assumptions; -- evidence needed to know the capability works. +* the business rule and its purpose; +* concrete examples and counterexamples; +* boundaries, exceptions, and failure outcomes; +* vocabulary that means the same thing to each participant; +* unanswered questions and assumptions; +* evidence needed to know the capability works. -The practice often called **Three Amigos** brings product or domain, development, and testing perspectives together. The name describes viewpoints, not a required meeting size or set of job titles. +The practice often called *Three Amigos* brings product or domain, development, and testing perspectives together. The name describes viewpoints, not a required meeting size or set of job titles. Each viewpoint asks different questions. Product clarifies value and rules, development exposes technical consequences, and testing searches for ambiguity, boundaries, and missing risks. @@ -71,25 +71,25 @@ Example Mapping is one lightweight way to organize the conversation into a story An executable scenario is most valuable after this conversation. Automating an unresolved example only makes uncertainty repeatable. -## Related practices overlap without becoming synonyms +== Related practices overlap without becoming synonyms -**BDD** is a family of discovery, formulation, and automation practices centered on behavior in shared language. +*BDD* is a family of discovery, formulation, and automation practices centered on behavior in shared language. -**Acceptance Test-Driven Development (ATDD)** clarifies acceptance examples collaboratively before implementation and uses them to guide development. +*Acceptance Test-Driven Development (ATDD)* clarifies acceptance examples collaboratively before implementation and uses them to guide development. -**Specification by Example** discovers and documents requirements through concrete examples, often retaining selected examples as living specifications. +*Specification by Example* discovers and documents requirements through concrete examples, often retaining selected examples as living specifications. All three can produce executable examples and improve the outer input to TDD. Their histories, communities, and emphases differ, so treating them as identical loses useful distinctions. None requires every conversational example to become an automated test. Automate examples when repeated evaluation and maintenance provide continuing value. -## Outer examples and inner TDD cycles +== Outer examples and inner TDD cycles A stakeholder-visible example may be too broad to provide feedback after every code edit. It can still guide a series of smaller cycles. -This pattern is called **double-loop TDD** or an outer acceptance loop with inner TDD cycles. +This pattern is called *double-loop TDD* or an outer acceptance loop with inner TDD cycles. -**Outside-in TDD** is a broader design direction that begins at a caller-visible boundary and discovers collaborators inward. It can use double loops, but the terms are not synonyms. +*Outside-in TDD* is a broader design direction that begins at a caller-visible boundary and discovers collaborators inward. It can use double loops, but the terms are not synonyms. The outer example states a capability at a caller-visible boundary. It helps select the next thin slice and shows when the assembled capability is complete. @@ -97,14 +97,16 @@ Inner Red–Green–Refactor cycles implement one observable behavior at a narro The loops answer different questions: -| Loop | Main question | Normal cadence | -| ----- | --------------------------------------------------------------------------------- | -------------------------------------- | -| Outer | Does the stakeholder-visible capability work through the selected broad boundary? | At meaningful slice or assembly points | -| Inner | Does the next focused behavior work, and can its design be improved safely? | Repeated during implementation | +[cols=",,",options="header",] +|=== +|Loop |Main question |Normal cadence +|Outer |Does the stakeholder-visible capability work through the selected broad boundary? |At meaningful slice or assembly points +|Inner |Does the next focused behavior work, and can its design be improved safely? |Repeated during implementation +|=== The outer example does not replace focused evidence. Inner checks do not prove assembly. Use both only when each answers a distinct risk. -## Keep the outer loop honest +== Keep the outer loop honest An outer example that represents unfinished behavior should not make the required shared suite permanently Red. @@ -114,21 +116,22 @@ When the outer example turns Green, run the standard lane and any thorough evide If an outer scenario takes days to complete, the inner loop still needs frequent closed cycles. A long-lived target is a planning aid, not permission to leave ordinary TDD cycles Red. -## A bounded collaborative flow +== A bounded collaborative flow For a new capability: -1. Identify the caller, desired outcome, and business purpose. -2. Discover rules, examples, counterexamples, and unanswered questions together. -3. Formulate one clear outer example when it will guide or verify the work. -4. Select a thin stakeholder-visible slice. -5. Implement it through closed inner Red–Green–Refactor cycles. -6. Revisit the outer example and any questions exposed by implementation. -7. Retain only evidence whose continuing value justifies its cost. +[arabic] +. Identify the caller, desired outcome, and business purpose. +. Discover rules, examples, counterexamples, and unanswered questions together. +. Formulate one clear outer example when it will guide or verify the work. +. Select a thin stakeholder-visible slice. +. Implement it through closed inner Red–Green–Refactor cycles. +. Revisit the outer example and any questions exposed by implementation. +. Retain only evidence whose continuing value justifies its cost. This flow is not mandatory ceremony. A clear, low-risk rule may need only a short conversation and a focused example. -## What BDD does not replace +== What BDD does not replace BDD does not replace product discovery, user research, architecture, exploratory testing, or specialized quality evaluation. @@ -140,12 +143,12 @@ Use collaboration where misunderstanding is a material risk. Use executable scen The applied chapters show how these ideas behave in greenfield, legacy, and organizational settings. The appendix points to broader practices that complement TDD without belonging to its core loop. -## Sources and further reading +== Sources and further reading -- Dan North, [Introducing BDD](https://dannorth.net/blog/introducing-bdd/) -- Dan North, [BDD is like TDD if...](https://dannorth.net/blog/bdd-is-like-tdd-if/) -- Cucumber, [Behaviour-Driven Development](https://cucumber.io/docs/bdd/) -- Agile Alliance, [Behavior-Driven Development](https://agilealliance.org/glossary/bdd/) -- Agile Alliance, [Acceptance Test-Driven Development](https://agilealliance.org/glossary/atdd/) -- Cucumber, [Example Mapping](https://cucumber.io/docs/bdd/example-mapping/) -- Steve Freeman and Nat Pryce, [Growing Object-Oriented Software, Guided by Tests](https://www.oreilly.com/library/view/growing-object-oriented-software/9780321574442/) +* Dan North, https://dannorth.net/blog/introducing-bdd/[Introducing BDD] +* Dan North, https://dannorth.net/blog/bdd-is-like-tdd-if/[BDD is like TDD if...] +* Cucumber, https://cucumber.io/docs/bdd/[Behaviour-Driven Development] +* Agile Alliance, https://agilealliance.org/glossary/bdd/[Behavior-Driven Development] +* Agile Alliance, https://agilealliance.org/glossary/atdd/[Acceptance Test-Driven Development] +* Cucumber, https://cucumber.io/docs/bdd/example-mapping/[Example Mapping] +* Steve Freeman and Nat Pryce, https://www.oreilly.com/library/view/growing-object-oriented-software/9780321574442/[Growing Object-Oriented Software, Guided by Tests] diff --git a/setup/optional-skills/knowledge/tdd/guide/06-applying-tdd-without-dogma.md b/setup/optional-skills/knowledge/tdd/guide/06-applying-tdd-without-dogma.adoc similarity index 73% rename from setup/optional-skills/knowledge/tdd/guide/06-applying-tdd-without-dogma.md rename to setup/optional-skills/knowledge/tdd/guide/06-applying-tdd-without-dogma.adoc index 8cec6ad..32a8327 100644 --- a/setup/optional-skills/knowledge/tdd/guide/06-applying-tdd-without-dogma.md +++ b/setup/optional-skills/knowledge/tdd/guide/06-applying-tdd-without-dogma.adoc @@ -1,10 +1,10 @@ -# Applying TDD without dogma += Applying TDD without dogma TDD exchanges effort now for earlier design feedback and reusable regression evidence. Whether that exchange is worthwhile depends on the behavior, risk, and cost of obtaining trustworthy feedback. This chapter examines those limits, then shows how entry into TDD differs for greenfield and legacy work. -## Where TDD tends to earn its cost +== Where TDD tends to earn its cost TDD has leverage when an executable oracle can express the next behavior, the code is likely to change, and early feedback can still influence its interface or responsibilities. @@ -14,7 +14,7 @@ The case becomes stronger when mistakes are expensive, discovered cases are easy TDD can also help when the final design is unclear but the next concrete behavior is known. The example constrains one decision without pretending to settle the whole architecture. -## Where another mode may be better +== Where another mode may be better TDD supplies less leverage when the immediate goal is learning rather than stable behavior. @@ -26,11 +26,11 @@ Stable code that is not changing may not justify retrospective tests. Coverage i Separate exploration from productionization. Explore cheaply, then preserve the parts that will survive behind explicit contracts and appropriate evidence. -## Serious objections and what survives them +== Serious objections and what survives them The practical question is not whether TDD is always good. It is which mechanisms help in this context, compared with the way the work would otherwise be performed. -### “The empirical evidence is mixed” +=== “The empirical evidence is mixed” Studies report varied effects on productivity and defects. Results depend on task, experience, cycle discipline, study design, and especially the comparison process. @@ -40,7 +40,7 @@ The narrower claim is mechanistic. Executable examples can clarify outcomes, sma Those mechanisms can be observed during work. Whether they repay their cost remains contextual. -### “TDD cannot design the architecture” +=== “TDD cannot design the architecture” Focused examples can reveal awkward interfaces and misplaced responsibilities. They cannot select a sound system architecture by themselves. @@ -48,7 +48,7 @@ Architecture also needs product discovery, quality-attribute analysis, integrati A walking skeleton can probe an initial architecture. Its result remains provisional rather than proof that the larger design is correct. -### “TDD damages design” +=== “TDD damages design” Tests can encourage interfaces, dependency injection, indirection, and fragmented responsibilities that the production problem never required. @@ -58,7 +58,7 @@ Writing a test first does not require mocking every collaborator. Run fast deter The test should make a production interface easier to use. It should not make every internal function replaceable. -### “Tests can also be wrong” +=== “Tests can also be wrong” A passing test proves only that the observed outcome matched its oracle. The requirement, example, test data, boundary, or implementation can still be wrong. @@ -66,7 +66,7 @@ Derive expected results independently, review consequential examples, and use se Delete obsolete tests and revise tests when the intended contract changes. Preserving an incorrect specification is not safety. -### “TDD slows obvious work down” +=== “TDD slows obvious work down” On an obvious, low-risk, short-lived change, formalizing every small step may cost more than it reveals. @@ -74,13 +74,13 @@ Use Obvious Implementation when the solution is clear. TDD still requires an obs The trade is effort now for possible savings in misunderstanding, debugging, regression detection, and future change. Reduce the ceremony when those benefits are unlikely. -### “Strict micro-cycles do not fit every change” +=== “Strict micro-cycles do not fit every change” Schema migrations, distributed protocols, and cross-cutting changes may need a coherent slice before any useful test can execute. Keep one behavioral decision in flight, but let the smallest coherent step match the system. “Small” is a feedback property, not a fixed line count or time limit. -### “Tests calcify the code” +=== “Tests calcify the code” A suite coupled to private methods, call order, framework details, or storage structure makes harmless refactoring expensive. @@ -88,7 +88,7 @@ That suite is a liability whether its tests were written before or after the imp Behavior-focused tests reduce this problem but do not remove maintenance cost. Intended behavior changes still require test changes. -### “TDD does not fit my domain” +=== “TDD does not fit my domain” Some domains lack stable exact outcomes. Aesthetic judgment, model quality, emergent behavior, and exploratory analysis may require review, experiments, datasets, or monitoring. @@ -98,15 +98,15 @@ An ML system may use ordinary tests for schemas and transformations while model The claim is never “test-drive everything.” -### “TDD gets credit for ordinary engineering practices” +=== “TDD gets credit for ordinary engineering practices” Test-first timing, small batches, automation, regression tests, and refactoring are separable mechanisms. Teams can use some without adopting canonical TDD. TDD packages them into a disciplined feedback sequence. Its value should be judged against the actual alternative, not against an imaginary process with no checking or design thought. -## Starting a new project +== Starting a new project -Begin with a **walking skeleton**: the thinnest end-to-end slice that proves the build, test runner, delivery path, and runtime can work together. +Begin with a *walking skeleton*: the thinnest end-to-end slice that proves the build, test runner, delivery path, and runtime can work together. The first behavior may be a service answering a health request. This removes delivery uncertainty before application logic accumulates. @@ -120,16 +120,17 @@ Keep required gates trustworthy. An unfinished outer example may remain pending, The greenfield sequence is therefore: -1. establish the smallest runnable build, test, and delivery skeleton; -2. select one stakeholder-visible capability; -3. frame one concrete behavior and its oracle; -4. drive its inner behavior through observed Red–Green–Refactor cycles; -5. verify the assembled slice with the appropriate outer evidence; -6. use what was learned to select the next slice. +[arabic] +. establish the smallest runnable build, test, and delivery skeleton; +. select one stakeholder-visible capability; +. frame one concrete behavior and its oracle; +. drive its inner behavior through observed Red–Green–Refactor cycles; +. verify the assembled slice with the appropriate outer evidence; +. use what was learned to select the next slice. Chapter 7 follows this sequence in a complete example. -## Starting in legacy code +== Starting in legacy code Legacy code may not permit a clean Red–Green–Refactor cycle immediately. Hard-wired dependencies and unknown behavior can prevent safe observation. @@ -143,28 +144,29 @@ That preparatory edit is risky because it lacks a test. Keep it mechanical, revi The legacy sequence is: -1. identify the intended change and the behavior at risk; -2. observe the current system through the best available boundary; -3. introduce a minimal seam first when no test can otherwise run; -4. characterize only behavior that must remain stable; -5. establish an observed Red for the new or corrected behavior; -6. make the smallest coherent Green change; -7. refactor under the improved evidence. +[arabic] +. identify the intended change and the behavior at risk; +. observe the current system through the best available boundary; +. introduce a minimal seam first when no test can otherwise run; +. characterize only behavior that must remain stable; +. establish an observed Red for the new or corrected behavior; +. make the smallest coherent Green change; +. refactor under the improved evidence. Questionable existing behavior needs domain review before it becomes a long-term specification. Approval testing can help capture complex output for that review. Adoption is incremental. Each touched area should leave behind clearer boundaries and more useful evidence than it had before. -## Coverage without coverage theatre +== Coverage without coverage theatre Coverage can reveal code the suite never executes. It cannot establish requirement completeness, oracle strength, or fault detection by itself. Use it to ask questions: -- Why is this changed branch unexercised? -- Is the code unreachable, low risk, or missing an example? -- Did configuration omit part of the suite? -- Does a regulatory objective require structural evidence? +* Why is this changed branch unexercised? +* Is the code unreachable, low risk, or missing an example? +* Did configuration omit part of the suite? +* Does a regulatory objective require structural evidence? Avoid using a percentage as an individual performance goal. Once rewards depend on the number, weak assertions and low-value tests become rational responses. @@ -172,23 +174,23 @@ Regulated or safety-critical work may require structural coverage objectives. Me Mutation testing can reveal assertions that miss selected faults. Inspect meaningful survivors instead of replacing one gamed score with another. -## Signals worth discussing +== Signals worth discussing No single metric represents test quality or engineering confidence. Useful observations include: -- time from a change to a trustworthy diagnosis; -- flaky-test rate and time spent investigating noise; -- escaped-defect themes and recurrence; -- change-failure and recovery patterns; -- mutation survivors in critical logic; -- maintenance cost of important suites; -- ease of changing a business rule without unrelated breakage. +* time from a change to a trustworthy diagnosis; +* flaky-test rate and time spent investigating noise; +* escaped-defect themes and recurrence; +* change-failure and recovery patterns; +* mutation survivors in critical logic; +* maintenance cost of important suites; +* ease of changing a business rule without unrelated breakage. Use these signals to investigate the system, not to rank individual developers. -## Test-suite health +== Test-suite health A retained suite is part of the production feedback system and requires maintenance. @@ -200,7 +202,7 @@ Broader tests should retain diagnostic artifacts. Without logs, identifiers, tra Delete redundant tests when cheaper evidence answers the same question. Rewrite tests that resist legitimate refactoring. Update examples when intended behavior changes. -## Introducing TDD to a team +== Introducing TDD to a team Adoption needs practice and feedback, not only a policy. @@ -210,22 +212,22 @@ Retrospectives should examine whether the loop reduced uncertainty and how much A practical team policy is: -- state the behavior and relevant risk before implementation; -- observe Red for new or corrected behavior with an executable oracle; -- select the shortest trustworthy feedback loop faithful to that risk; -- use broader evidence for infrastructure, compatibility, and deployed behavior; -- introduce doubles and seams only when control or observation justifies them; -- treat coverage and mutation results as prompts for investigation; -- maintain fast, standard, and thorough feedback lanes; -- reconsider tests whose maintenance cost exceeds the evidence they provide. - -## Sources and further reading - -- David Heinemeier Hansson, [TDD is dead. Long live testing.](https://dhh.dk/2014/tdd-is-dead-long-live-testing.html) -- Kent Beck, [Canon TDD](https://newsletter.kentbeck.com/p/canon-tdd) -- Michael Feathers, *Working Effectively with Legacy Code* -- Michael Feathers, [The Seam Model](https://www.informit.com/articles/article.aspx?p=359417&seqNum=2) -- Martin Fowler, [Test Coverage](https://martinfowler.com/bliki/TestCoverage.html) -- Freeman and Pryce, [Growing Object-Oriented Software, Guided by Tests](https://www.oreilly.com/library/view/growing-object-oriented-software/9780321574442/) -- Rafique and Mišić, [The Effects of Test-Driven Development on External Quality and Productivity](https://doi.org/10.1109/TSE.2012.28) -- Munir et al., [Considering rigor and relevance when evaluating test driven development](https://doi.org/10.1016/j.infsof.2014.01.001) +* state the behavior and relevant risk before implementation; +* observe Red for new or corrected behavior with an executable oracle; +* select the shortest trustworthy feedback loop faithful to that risk; +* use broader evidence for infrastructure, compatibility, and deployed behavior; +* introduce doubles and seams only when control or observation justifies them; +* treat coverage and mutation results as prompts for investigation; +* maintain fast, standard, and thorough feedback lanes; +* reconsider tests whose maintenance cost exceeds the evidence they provide. + +== Sources and further reading + +* David Heinemeier Hansson, https://dhh.dk/2014/tdd-is-dead-long-live-testing.html[TDD is dead. Long live testing.] +* Kent Beck, https://newsletter.kentbeck.com/p/canon-tdd[Canon TDD] +* Michael Feathers, _Working Effectively with Legacy Code_ +* Michael Feathers, https://www.informit.com/articles/article.aspx?p=359417&seqNum=2[The Seam Model] +* Martin Fowler, https://martinfowler.com/bliki/TestCoverage.html[Test Coverage] +* Freeman and Pryce, https://www.oreilly.com/library/view/growing-object-oriented-software/9780321574442/[Growing Object-Oriented Software, Guided by Tests] +* Rafique and Mišić, https://doi.org/10.1109/TSE.2012.28[The Effects of Test-Driven Development on External Quality and Productivity] +* Munir et al., https://doi.org/10.1016/j.infsof.2014.01.001[Considering rigor and relevance when evaluating test driven development] diff --git a/setup/optional-skills/knowledge/tdd/guide/07-worked-examples.md b/setup/optional-skills/knowledge/tdd/guide/07-worked-examples.adoc similarity index 85% rename from setup/optional-skills/knowledge/tdd/guide/07-worked-examples.md rename to setup/optional-skills/knowledge/tdd/guide/07-worked-examples.adoc index 815983c..e4102c0 100644 --- a/setup/optional-skills/knowledge/tdd/guide/07-worked-examples.md +++ b/setup/optional-skills/knowledge/tdd/guide/07-worked-examples.adoc @@ -1,34 +1,36 @@ -# Worked examples += Worked examples These examples show decisions unfolding over time. The code is illustrative Python, but the sequence applies across stacks. Each complete example records the baseline, expected behavior, observed Red, coherent Green, refactoring decision, and broader verification. -Terms are defined in the [glossary](appendices/glossary.md). +Terms are defined in the xref:appendices/glossary.adoc[glossary]. The example repository uses illustrative lane commands: -```text +[source,text] +---- fast/local: pytest tests/unit -q standard/change-validation: pytest tests/unit tests/component tests/integration -q thorough/system-validation: pytest -q -``` +---- Real projects should use their own measured commands and gates. -## Complete example 1: a greenfield delivery quote +== Complete example 1: a greenfield delivery quote A new service has a build and test command but no proven runtime or delivery path. -### Establish the walking skeleton +=== Establish the walking skeleton The team first verifies the known starting state: -```text +[source,text] +---- $ python -m compileall -q src $ pytest --version pytest 9.x -``` +---- The first component example requests `/health` and expects status 200. It initially receives 404, showing that the request reaches the application but the health capability is absent. @@ -38,95 +40,103 @@ The skeleton now proves build, test, request routing, deployment, and runtime co The first stakeholder-visible slice is a delivery quote. Orders at or above EUR 50 receive free standard delivery. The price below that threshold is still an open policy question. -### Frame the slice +=== Frame the slice The team writes a provisional test list: -- EUR 50 receives free delivery; -- determine the price below EUR 50; -- an amount above EUR 50 receives free delivery; -- invalid negative totals are rejected; -- the HTTP endpoint returns the calculated quote. +* EUR 50 receives free delivery; +* determine the price below EUR 50; +* an amount above EUR 50 receives free delivery; +* invalid negative totals are rejected; +* the HTTP endpoint returns the calculated quote. Only the first item becomes executable. The others remain reminders that may change after each cycle. The outer HTTP example is recorded as pending while the domain rule is developed through faster inner cycles: -```python +[source,python] +---- @pytest.mark.skip(reason="delivery quote is in progress") def test_quote_endpoint_returns_free_delivery_at_threshold(client): response = client.post("/quotes", json={"order_total": "50.00"}) assert response.status_code == 200 assert response.json() == {"standard_delivery_fee": "0.00"} -``` +---- -### Establish the baseline +=== Establish the baseline The health and test-runner checks pass: -```text +[source,text] +---- $ pytest -q 1 passed, 1 skipped -``` +---- There is no pre-existing failure that could be confused with the new Red. -### First Red +=== First Red The first focused example states the inclusive threshold: -```python +[source,python] +---- def test_standard_delivery_is_free_at_fifty_euros(): assert standard_delivery_fee(Decimal("50.00")) == Decimal("0.00") -``` +---- A missing symbol is only an intermediate Red. After adding the callable surface without the behavior, the test reaches the contract failure: -```text +[source,text] +---- FAILED: expected Decimal("0.00"), received None -``` +---- The failure is reproducible, attributable to the missing rule, and demonstrates that the assertion can detect its absence. -### First Green +=== First Green The implementation supports the agreed range and refuses to invent the unresolved price: -```python +[source,python] +---- def standard_delivery_fee(order_total): if order_total >= Decimal("50.00"): return Decimal("0.00") raise QuotePolicyUndefined() -``` +---- The focused test passes. This implementation is intentionally narrow, but it does not violate or silently invent an established requirement. -### Second Red +=== Second Red The product owner now decides that smaller orders cost EUR 5. The next example makes that newly agreed behavior executable: -```python +[source,python] +---- def test_standard_delivery_costs_five_euros_below_threshold(): assert standard_delivery_fee(Decimal("49.99")) == Decimal("5.00") -``` +---- The test fails with `QuotePolicyUndefined`. The new example now justifies behavior below the threshold. -### Second Green +=== Second Green -```python +[source,python] +---- def standard_delivery_fee(order_total): if order_total >= Decimal("50.00"): return Decimal("0.00") return Decimal("5.00") -``` +---- Both examples pass. The code handles only behavior supported by the current examples. The pending outer example can now include the newly agreed paid-delivery outcome. It is revised while still pending: -```python +[source,python] +---- @pytest.mark.skip(reason="delivery quote is in progress") def test_quote_endpoint_returns_standard_delivery_fees(client): free = client.post("/quotes", json={"order_total": "50.00"}) @@ -135,9 +145,9 @@ def test_quote_endpoint_returns_standard_delivery_fees(client): assert free.status_code == paid.status_code == 200 assert free.json() == {"standard_delivery_fee": "0.00"} assert paid.json() == {"standard_delivery_fee": "5.00"} -``` +---- -### Review the remaining examples +=== Review the remaining examples An example above the threshold passes immediately because the current comparison already supports it. The team confirms that it reaches production code and retains it because the inclusive range is part of the contract. @@ -145,21 +155,23 @@ This is useful regression evidence, but it is not presented as a Red–Green cyc The team then agrees that negative totals are invalid: -```python +[source,python] +---- def test_standard_delivery_rejects_negative_total(): with pytest.raises(InvalidOrderTotal): standard_delivery_fee(Decimal("-0.01")) -``` +---- The test fails because the current implementation returns EUR 5. That observed failure establishes the next Red. The smallest coherent Green adds the validation guard. All threshold and invalid-input examples pass. -### Refactor while Green +=== Refactor while Green The literals now carry domain meaning, so they are named without changing behavior: -```python +[source,python] +---- FREE_DELIVERY_THRESHOLD = Decimal("50.00") STANDARD_DELIVERY_FEE = Decimal("5.00") @@ -170,20 +182,21 @@ def standard_delivery_fee(order_total): if order_total >= FREE_DELIVERY_THRESHOLD: return Decimal("0.00") return STANDARD_DELIVERY_FEE -``` +---- The focused tests remain Green after the structural change. -### Complete the vertical slice +=== Complete the vertical slice The pending marker is removed. The outer example now runs and receives 404 because the quote route does not exist. The route is added as a thin adapter over the tested rule: -```text +[source,text] +---- $ pytest tests/component/test_quotes_api.py -q 1 passed -``` +---- The contrasting requests confirm parsing, routing, serialization, and results consistent with the domain rule. They do not prove whether the adapter delegates or duplicates that rule. @@ -193,46 +206,50 @@ Before handoff, the standard command shown above passes. The thorough `pytest -q The next slice can now be selected from what the team learned. -## Complete example 2: correcting an existing defect +== Complete example 2: correcting an existing defect An existing percentage parser accepts negative discounts. Its current positive and zero-value tests pass. -### Establish the baseline +=== Establish the baseline -```text +[source,text] +---- $ pytest tests/unit/test_discounts.py -q 6 passed -``` +---- -### Establish Red +=== Establish Red -```python +[source,python] +---- def test_rejects_negative_percentage(): with pytest.raises(InvalidDiscountError): parse_discount("-10%") -``` +---- The observed result is: -```text +[source,text] +---- FAILED: did not raise InvalidDiscountError -``` +---- This is the intended Red. A parsing error, missing fixture, or unrelated failure would not demonstrate the defect. -### Reach Green +=== Reach Green -```python +[source,python] +---- def parse_discount(text): value = parse_percentage(text) if value < 0: raise InvalidDiscountError("discount must not be negative") return value -``` +---- The new test and the neighboring parser tests pass. -### Refactor and verify +=== Refactor and verify Validation already exists in another percentage-based input. Extracting a shared domain validator removes that duplication while all examples remain Green. @@ -242,34 +259,36 @@ No browser test is added because browser behavior was not part of the defect. The correction is test-first even though the original parser was not developed through TDD. -## Complete example 3: changing an existing contract +== Complete example 3: changing an existing contract An order may currently be reopened at any time after cancellation. The clarified requirement permits reopening through exactly ten minutes, but not later. This is an intentional behavior change. Existing tests are specifications to revise, not immutable obstacles. -### Establish the baseline +=== Establish the baseline The existing reopening examples pass. The team confirms the new time limit with the product owner and chooses the public `reopen` operation as the observation boundary. -### Revise retained evidence and establish Red +=== Revise retained evidence and establish Red The former unconditional example is narrowed to the still-supported behavior: -```python +[source,python] +---- def test_cancelled_order_can_be_reopened_at_exactly_ten_minutes(): order = cancelled_order(cancelled_at=instant("10:00")) order.reopen(now=instant("10:10")) assert order.status == OrderStatus.OPEN -``` +---- This example already passes because it describes a subset of existing behavior. It clarifies the retained contract but does not establish Red. The next example expresses the unsupported restriction: -```python +[source,python] +---- def test_cancelled_order_cannot_be_reopened_after_ten_minutes(): order = cancelled_order(cancelled_at=instant("10:00")) @@ -277,22 +296,23 @@ def test_cancelled_order_cannot_be_reopened_after_ten_minutes(): order.reopen(now=instant("10:10:00.001")) assert order.status == OrderStatus.CANCELLED -``` +---- The test fails because no exception is raised. That observed failure establishes Red for the changed contract. -### Reach Green +=== Reach Green -```python +[source,python] +---- def reopen(self, now): if now - self.cancelled_at > timedelta(minutes=10): raise ReopenWindowExpired() self.status = OrderStatus.OPEN -``` +---- Both the retained and restricted examples pass. -### Refactor and verify +=== Refactor and verify The ten-minute duration becomes a named policy value. Cancellation behavior unrelated to reopening remains unchanged. @@ -300,46 +320,49 @@ Focused order tests and the standard order-workflow suite pass. Published client This example shows why the runtime instruction must say “add or revise a test,” rather than only “add a test.” -## Complete example 4: entering difficult legacy code +== Complete example 4: entering difficult legacy code A legacy invoice service constructs a tax client internally. Local tests cannot substitute it, and the real client requires credentials available only in the deployed test environment. The requested change is to exempt zero-value invoices from the tax request. Existing non-zero invoice behavior must remain stable. -### Establish the best available baseline +=== Establish the best available baseline The focused local test cannot run because construction reaches the credentialed provider. That limitation is recorded instead of being called Red. An existing smoke check runs in the credentialed deployed test environment and passes: -```text +[source,text] +---- $ pytest tests/system/test_invoice_smoke.py -q 1 passed -``` +---- The team will rerun this check immediately after the unprotected dependency break. -### Identify the risk +=== Identify the risk No useful characterization test can run while construction is hard-wired. The first step is therefore a minimal dependency break, not a fictional Red. The constructor is changed to accept an optional tax client while preserving the production default: -```python +[source,python] +---- class InvoiceService: def __init__(self, tax_client=None): self.tax_client = ( tax_client if tax_client is not None else RealTaxClient() ) -``` +---- This edit is mechanical but not protected by a new test. It receives careful review, then the existing system smoke check passes again. -### Characterize behavior to preserve +=== Characterize behavior to preserve A deterministic fake now makes current non-zero behavior observable: -```python +[source,python] +---- def test_non_zero_invoice_requests_current_tax_amount(): taxes = RecordingTaxClient(amount=Decimal("2.00")) service = InvoiceService(tax_client=taxes) @@ -348,13 +371,14 @@ def test_non_zero_invoice_requests_current_tax_amount(): assert invoice.total == Decimal("12.00") assert taxes.requested_subtotals == [Decimal("10.00")] -``` +---- The characterization test passes. It records the behavior at risk; it is not the Red for the requested change. -### Establish Red for the change +=== Establish Red for the change -```python +[source,python] +---- def test_zero_value_invoice_does_not_request_tax(): taxes = RecordingTaxClient(amount=Decimal("0.00")) service = InvoiceService(tax_client=taxes) @@ -365,49 +389,52 @@ def test_zero_value_invoice_does_not_request_tax(): assert invoice.tax == Decimal("0.00") assert invoice.total == Decimal("0.00") assert taxes.requested_subtotals == [] -``` +---- The test fails because the fake records a request for `Decimal("0.00")`. -### Reach Green +=== Reach Green The coherent early return still creates the required invoice: -```python +[source,python] +---- def create_invoice(self, subtotal): if subtotal == Decimal("0.00"): return Invoice(subtotal=subtotal, tax=Decimal("0.00")) tax = self.tax_client.calculate(subtotal) return Invoice(subtotal=subtotal, tax=tax) -``` +---- The outcome assertions prevent an implementation that merely skips all work. The interaction assertion proves that zero-value invoices avoid the external request. -### Refactor and verify +=== Refactor and verify Only then is tax-request construction extracted into a small private operation. Public invoice behavior remains unchanged while the tests stay Green. The developer runs `pytest tests/unit/invoicing tests/contract/tax_provider -q`. The fake proves what the service requested; the contract check covers provider compatibility, not real settlement. -## Complete example 5: when integration evidence is the correct Red +== Complete example 5: when integration evidence is the correct Red A SQL Server application requires usernames to be unique under its production collation. An in-memory collection cannot supply that oracle. The test uses a disposable SQL Server instance with production-relevant schema and collation. -### Establish the baseline +=== Establish the baseline The existing registration integration tests pass against the disposable engine: -```text +[source,text] +---- $ pytest tests/integration/test_registration_sqlserver.py -q 4 passed -``` +---- -### Establish Red +=== Establish Red -```python +[source,python] +---- def test_username_constraint_uses_production_collation(sql_user_repository): sql_user_repository.insert(username="Alice") @@ -415,46 +442,48 @@ def test_username_constraint_uses_production_collation(sql_user_repository): sql_user_repository.insert(username="alice") assert sql_user_repository.count_matching("alice") == 1 -``` +---- Both inserts succeed and the count becomes two. The new test is the only failure, and it observes the database constraint and persisted state rather than only an application return value. -### Reach Green +=== Reach Green A production-relevant unique index is added. A separate application integration example verifies translation of that constraint violation into the duplicate-user result. The new example and neighboring SQL Server tests pass. -### Refactor and verify +=== Refactor and verify The constraint-to-domain translation is extracted behind one named mapping operation while the integration suite remains Green. The developer runs: -```text +[source,text] +---- $ pytest tests/unit/registration tests/integration/test_registration_sqlserver.py -q 18 passed -``` +---- The migration test also verifies clean installation and upgrade from the previous schema. A focused test may support diagnosis, but it cannot replace the database evidence. This slower test is the correct Red because only it faithfully exercises the risk. Runtime speed is not allowed to select an irrelevant boundary. -## Compact decision examples +== Compact decision examples The remaining examples illustrate choices rather than complete development sequences. -### Requesting an external effect +=== Requesting an external effect A focused test can prove that checkout requested a charge: -```python +[source,python] +---- payment_gateway.charge.assert_called_once_with(order.total) -``` +---- It does not prove settlement. Provider contracts, sandbox integration, and operational reconciliation answer different questions. -### Consumer-driven contract evidence +=== Consumer-driven contract evidence A consumer records the requests and responses it relies on. Provider verification replays those interactions against the provider build. @@ -462,30 +491,31 @@ This detects exercised incompatibility without proving authorization, database b Use it when independently changing consumers and providers need compatibility feedback before release. -### E2E evidence for a deployed path +=== E2E evidence for a deployed path One checkout E2E test can cover routing, browser behavior, deployed configuration, and service wiring. Keep discount, tax, and validation partitions in faster tests. Capture traces, scoped logs, and screenshots so a broad failure remains diagnosable. -### Controlled nondeterminism +=== Controlled nondeterminism A retry policy with jitter can accept a seeded random source and expose bounds: -```python +[source,python] +---- delay = retry_delay(attempt=3, random_source=seeded_random) assert minimum <= delay <= maximum -``` +---- Distribution quality still needs repeated statistical evaluation. One exact example is not a sufficient oracle. -### Test-after evidence +=== Test-after evidence A black-box test written after implementation can still provide useful specification and regression evidence. It did not supply design feedback during the original implementation. Test timing and test quality are separate judgments. -### Testability rather than mockability +=== Testability rather than mockability Code that constructs a payment SDK, reads the system clock, and returns nothing is hard to control and observe. @@ -493,14 +523,14 @@ Move SDK construction to the application’s startup wiring, often called the co Keep deterministic owned helpers together when that gives clearer behavior tests. The number of interfaces or mocks is not a quality measure. -## What the examples demonstrate +== What the examples demonstrate The examples differ in scope, setup, and speed, but keep the same evidence discipline: -- know the starting state; -- choose a faithful boundary and independent oracle; -- observe meaningful Red for new or corrected behavior; -- reach Green without weakening the evidence; -- refactor only while behavior remains Green; -- run broader evidence according to risk; -- report what remains unverified. +* know the starting state; +* choose a faithful boundary and independent oracle; +* observe meaningful Red for new or corrected behavior; +* reach Green without weakening the evidence; +* refactor only while behavior remains Green; +* run broader evidence according to risk; +* report what remains unverified. diff --git a/setup/optional-skills/knowledge/tdd/guide/appendices/adjacent-practices-and-further-reading.adoc b/setup/optional-skills/knowledge/tdd/guide/appendices/adjacent-practices-and-further-reading.adoc new file mode 100644 index 0000000..71408f6 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/guide/appendices/adjacent-practices-and-further-reading.adoc @@ -0,0 +1,191 @@ += Adjacent practices and further reading + +TDD is one feedback workflow, not a complete engineering system. This appendix points to practices that answer nearby questions but are too broad for the core reading path. + +Each entry explains its question, connection to TDD, and trigger for deeper study. The links are starting points, not a required curriculum. + +== Test doubles and testability + +*Question:* How can a test control inputs or observe outcomes when a collaborator is slow, nondeterministic, destructive, or outside the team's control? + +*Relationship to TDD:* Dummies, stubs, spies, mocks, and fakes can make a boundary controllable. They do not prove the real external effect, and testability is broader than mockability. + +*Investigate when:* Time, randomness, networks, costly infrastructure, or separately governed systems obstruct useful feedback. Also investigate when interaction-heavy tests make safe refactoring difficult. + +*Start with:* Gerard Meszaros, https://xunitpatterns.com/[xUnit Test Patterns]; Martin Fowler, https://martinfowler.com/articles/mocksArentStubs.html[Mocks Aren't Stubs]. + +== TDD schools and working styles + +*Question:* Where should design pressure enter, and when should tests use real owned collaborators rather than substitutes? + +*Relationship to TDD:* Classicist TDD commonly uses real owned collaborators. Mockist TDD uses interaction expectations to discover roles. Inside-out and outside-in describe another, related axis. + +*Investigate when:* A team disagrees about isolation, test boundaries, or whether interfaces discovered through mocks clarify the domain. Judge outcomes, not school labels. + +*Start with:* Fowler, https://martinfowler.com/articles/mocksArentStubs.html[Mocks Aren't Stubs]. + +Also see Freeman and Pryce, https://www.oreilly.com/library/view/growing-object-oriented-software/9780321574442/[Growing Object-Oriented Software]. + +== Test portfolio heuristics + +*Question:* How should a project distribute focused, integration, and E2E evidence so feedback remains useful and affordable? + +*Relationship to TDD:* The pyramid, trophy, and honeycomb visualize cost or fidelity preferences. They do not determine the correct Red or replace risk analysis. + +*Investigate when:* The suite is slow, fragile, duplicative, or blind to important integration risks. Use measurements and unique questions rather than target proportions. + +*Start with:* Fowler, https://martinfowler.com/articles/practical-test-pyramid.html[The Practical Test Pyramid]; Kent C. Dodds, https://kentcdodds.com/blog/the-testing-trophy-and-testing-classifications[The Testing Trophy]. + +== Risk-based testing + +*Question:* Which behaviors and conditions deserve the most evidence when time and attention are limited? + +*Relationship to TDD:* Risk helps select the next behavior, observation boundary, and verification depth. Neither every function nor every data transformation deserves equal treatment. + +*Investigate when:* Impact, failure probability, change frequency, complexity, poor observability, or regulatory exposure varies materially across the system. + +*Start with:* ISO/IEC/IEEE 29119-2, https://www.iso.org/standard/79428.html[Test processes]; ISTQB, https://www.istqb.org/certifications/certified-tester-advanced-level-test-management-ctal-tm-v3-0/[Advanced Test Management]. + +== Exploratory testing + +*Question:* What important behavior or risk has nobody encoded as an expectation yet? + +*Relationship to TDD:* Automated examples check known claims repeatedly. Exploration combines learning, test design, and execution to discover surprises, usability problems, and new questions. + +*Investigate when:* The domain is unfamiliar, behavior is visual or experiential, incidents reveal unknown risks, or scripted checks pass while confidence remains low. + +*Start with:* Cem Kaner and James Bach, https://kaner.com/pdfs/ETatQAI.pdf[The Nature of Exploratory Testing]; James Bach, https://www.satisfice.com/download/exploratory-testing-explained[Exploratory Testing Explained]. + +== Property-based testing + +*Question:* Does an invariant hold across a generated range of inputs rather than only selected examples? + +*Relationship to TDD:* Properties can follow examples or drive a cycle when a trustworthy invariant is known. Generator and oracle quality still determine the evidence. + +*Investigate when:* Parsers, transformations, algebraic rules, protocols, or state machines have large input spaces and useful invariants. + +*Start with:* Koen Claessen and John Hughes, https://www.cs.tufts.edu/~nr/cs257/archive/john-hughes/quick.pdf[QuickCheck]. + +== Metamorphic testing + +*Question:* When an exact expected output is unavailable, should related executions still have a known relationship? + +*Relationship to TDD:* A metamorphic relation can serve as an oracle. It complements concrete examples when calculating one exact answer independently is difficult. + +*Investigate when:* Search, simulation, scientific computing, optimization, and ML systems have an oracle problem but support meaningful transformations or relations. + +*Start with:* Tsong Yueh Chen et al., https://doi.org/10.1145/3143561[Metamorphic Testing: A Review of Challenges and Opportunities]. + +== Model-based testing + +*Question:* Do generated action sequences and state transitions agree with an explicit behavioral model? + +*Relationship to TDD:* A model supplies an oracle across sequences that individual examples may miss. Creating and maintaining it is a separate design investment. + +*Investigate when:* Stateful workflows, protocols, devices, or user interfaces have many valid sequences and transition rules. + +*Start with:* Mark Utting, Alexander Pretschner, and Bruno Legeard, https://doi.org/10.1002/stvr.456[A Taxonomy of Model-Based Testing Approaches]. + +== Mutation testing + +*Question:* Would the retained tests detect selected plausible changes to production code? + +*Relationship to TDD:* Mutation probes the suite's ability to discriminate behavior. A mutation score is evidence to interpret, not a universal quality target. + +*Investigate when:* Coverage is high but assertions may be weak, critical rules deserve an evidence audit, or a team wants to find untested decision outcomes. + +*Start with:* Yue Jia and Mark Harman, https://doi.org/10.1109/TSE.2010.62[An Analysis and Survey of Mutation Testing]; Stryker, https://stryker-mutator.io/docs/mutation-testing-elements/[Mutation testing elements]. + +== Approval and snapshot testing + +*Question:* How can a team review and retain complex output when many small assertions would hide the meaningful whole? + +*Relationship to TDD:* An approved baseline can become an oracle for new or existing behavior. Snapshot storage is a mechanism; deliberate human review gives a change meaning. + +*Investigate when:* Reports, rendered output, serialized structures, compiler output, or legacy behavior are costly to express with focused assertions. + +*Start with:* ApprovalTests, https://approvaltestscpp.readthedocs.io/en/latest/generated_docs/ApprovalTestingConcept.html[Approval Testing]; Jest, https://jestjs.io/docs/snapshot-testing[Snapshot Testing]. + +== Static analysis, type systems, and formal methods + +*Question:* Which defects can be rejected or which critical properties can be justified without executing example-based tests? + +*Relationship to TDD:* Types and static checks shorten feedback for classes of invalid programs. Model checking and proof can establish properties that sampled executions cannot. + +*Investigate when:* Invalid states can be excluded by design, concurrency creates large state spaces, or failure impact justifies stronger assurance than examples provide. + +*Start with:* Benjamin Pierce, https://www.cis.upenn.edu/~bcpierce/tapl/[Types and Programming Languages resources]; Leslie Lamport, https://lamport.azurewebsites.net/tla/tla.html[The TLA{plus} Home Page]. + +== Quality attributes + +*Question:* Does the system remain acceptable under security, privacy, accessibility, performance, resilience, concurrency, compatibility, and resource constraints? + +*Relationship to TDD:* Some attributes can be driven by executable examples. Most also require threat models, workloads, fault injection, audits, specialist tools, or statistical evidence. + +*Investigate when:* The attribute affects users, legal duties, safety, cost, service objectives, or architecture. Define measurable quality scenarios rather than calling the concern “non-functional.” + +*Start with:* https://owasp.org/www-project-application-security-verification-standard/[OWASP ASVS]; https://www.w3.org/TR/WCAG22/[W3C WCAG]; Google, https://sre.google/books/[Site Reliability Engineering]. + +For privacy engineering, also see NIST, https://www.nist.gov/privacy-framework[Privacy Framework]. Select domain standards with qualified specialists where legal or safety obligations apply. + +== Monitoring, observability, and reconciliation + +*Question:* What can be learned only from the system operating with real traffic, dependencies, data, and failure modes? + +*Relationship to TDD:* Operational feedback detects outcomes pre-release checks cannot reproduce. It complements rather than excuses missing development evidence. + +*Investigate when:* Systems are distributed, depend on third parties, degrade gradually, process asynchronous work, or require confirmation of real business outcomes. + +*Start with:* Google SRE, https://sre.google/sre-book/monitoring-distributed-systems/[Monitoring Distributed Systems]; OpenTelemetry, https://opentelemetry.io/docs/concepts/observability-primer/[Observability primer]. + +== ML and LLM evaluation + +*Question:* How well does a probabilistic model behave across representative tasks, populations, adversarial inputs, and changing production conditions? + +*Relationship to TDD:* Test deterministic preprocessing, schemas, tool calls, and control flow normally. Model quality usually needs datasets, metrics, repeated trials, human judgment, and monitoring. + +*Investigate when:* Outputs vary, exact strings are not the real requirement, model or prompt changes shift behavior, or safety and fairness depend on a distribution of outcomes. + +*Start with:* NIST, https://www.nist.gov/itl/ai-risk-management-framework[AI Risk Management Framework]; NIST, https://doi.org/10.6028/NIST.AI.600-1[Generative AI Profile]. + +== Regulatory and safety structural coverage + +*Question:* Which prescribed evidence, traceability, independence, and structural coverage objectives apply to this product and assurance level? + +*Relationship to TDD:* TDD can contribute executable requirements and regression evidence. It does not replace mandated processes, reviews, traceability, or coverage analysis. + +*Investigate when:* Software affects aviation, automotive, medical, rail, industrial, financial, or other regulated and safety-critical domains. Engage assurance specialists early. + +*Start with:* FAA, https://www.faa.gov/documentLibrary/media/Advisory_Circular/AC_20-115D.pdf[Advisory Circular 20-115D]; ISO, https://www.iso.org/standard/68388.html[ISO 26262-6 software development]. + +Do not transfer a coverage target from one standard, assurance level, or jurisdiction to another. Determine the exact governing obligations for the system. + +== Continuous integration and delivery + +*Question:* Does work remain integrated, deployable, and evaluable as contributions and environments change? + +*Relationship to TDD:* CI extends the local feedback loop to the team. Delivery automation tests packaging and deployment paths that focused development checks cannot prove. + +*Investigate when:* More than one change stream exists, integration is delayed, releases are risky, or environment drift weakens confidence. + +*Start with:* Martin Fowler, https://martinfowler.com/articles/continuousIntegration.html[Continuous Integration]; Jez Humble, https://continuousdelivery.com/[Continuous Delivery]. + +== Architecture and walking skeletons + +*Question:* Which system boundaries, runtime responsibilities, and delivery paths must work together before local design choices can succeed? + +*Relationship to TDD:* Tests provide local design feedback but do not automatically grow a sound system architecture. A walking skeleton probes a provisional end-to-end structure early. + +*Investigate when:* Starting a system, crossing trust or deployment boundaries, making irreversible technology choices, or changing system-wide qualities. + +*Start with:* Freeman and Pryce, https://www.oreilly.com/library/view/growing-object-oriented-software/9780321574442/ch10.html[Walking Skeleton]. + +Also see Michael Nygard, https://www.cognitect.com/blog/2011/11/15/documenting-architecture-decisions[Documenting Architecture Decisions]. + +Use spikes, prototypes, architecture decision records, and quality scenarios when uncertainty requires them. Keep the probe provisional; it is not permission for comprehensive design up front. + +== How to use this appendix + +Return to the core TDD loop when the current question is behavior that can be expressed through a trustworthy executable example. + +Follow an adjacent practice when it answers a different material question. Combine evidence deliberately, and state what each result does and does not establish. diff --git a/setup/optional-skills/knowledge/tdd/guide/appendices/adjacent-practices-and-further-reading.md b/setup/optional-skills/knowledge/tdd/guide/appendices/adjacent-practices-and-further-reading.md deleted file mode 100644 index 12600a2..0000000 --- a/setup/optional-skills/knowledge/tdd/guide/appendices/adjacent-practices-and-further-reading.md +++ /dev/null @@ -1,191 +0,0 @@ -# Adjacent practices and further reading - -TDD is one feedback workflow, not a complete engineering system. This appendix points to practices that answer nearby questions but are too broad for the core reading path. - -Each entry explains its question, connection to TDD, and trigger for deeper study. The links are starting points, not a required curriculum. - -## Test doubles and testability - -**Question:** How can a test control inputs or observe outcomes when a collaborator is slow, nondeterministic, destructive, or outside the team's control? - -**Relationship to TDD:** Dummies, stubs, spies, mocks, and fakes can make a boundary controllable. They do not prove the real external effect, and testability is broader than mockability. - -**Investigate when:** Time, randomness, networks, costly infrastructure, or separately governed systems obstruct useful feedback. Also investigate when interaction-heavy tests make safe refactoring difficult. - -**Start with:** Gerard Meszaros, [xUnit Test Patterns](https://xunitpatterns.com/); Martin Fowler, [Mocks Aren't Stubs](https://martinfowler.com/articles/mocksArentStubs.html). - -## TDD schools and working styles - -**Question:** Where should design pressure enter, and when should tests use real owned collaborators rather than substitutes? - -**Relationship to TDD:** Classicist TDD commonly uses real owned collaborators. Mockist TDD uses interaction expectations to discover roles. Inside-out and outside-in describe another, related axis. - -**Investigate when:** A team disagrees about isolation, test boundaries, or whether interfaces discovered through mocks clarify the domain. Judge outcomes, not school labels. - -**Start with:** Fowler, [Mocks Aren't Stubs](https://martinfowler.com/articles/mocksArentStubs.html). - -Also see Freeman and Pryce, [Growing Object-Oriented Software](https://www.oreilly.com/library/view/growing-object-oriented-software/9780321574442/). - -## Test portfolio heuristics - -**Question:** How should a project distribute focused, integration, and E2E evidence so feedback remains useful and affordable? - -**Relationship to TDD:** The pyramid, trophy, and honeycomb visualize cost or fidelity preferences. They do not determine the correct Red or replace risk analysis. - -**Investigate when:** The suite is slow, fragile, duplicative, or blind to important integration risks. Use measurements and unique questions rather than target proportions. - -**Start with:** Fowler, [The Practical Test Pyramid](https://martinfowler.com/articles/practical-test-pyramid.html); Kent C. Dodds, [The Testing Trophy](https://kentcdodds.com/blog/the-testing-trophy-and-testing-classifications). - -## Risk-based testing - -**Question:** Which behaviors and conditions deserve the most evidence when time and attention are limited? - -**Relationship to TDD:** Risk helps select the next behavior, observation boundary, and verification depth. Neither every function nor every data transformation deserves equal treatment. - -**Investigate when:** Impact, failure probability, change frequency, complexity, poor observability, or regulatory exposure varies materially across the system. - -**Start with:** ISO/IEC/IEEE 29119-2, [Test processes](https://www.iso.org/standard/79428.html); ISTQB, [Advanced Test Management](https://www.istqb.org/certifications/certified-tester-advanced-level-test-management-ctal-tm-v3-0/). - -## Exploratory testing - -**Question:** What important behavior or risk has nobody encoded as an expectation yet? - -**Relationship to TDD:** Automated examples check known claims repeatedly. Exploration combines learning, test design, and execution to discover surprises, usability problems, and new questions. - -**Investigate when:** The domain is unfamiliar, behavior is visual or experiential, incidents reveal unknown risks, or scripted checks pass while confidence remains low. - -**Start with:** Cem Kaner and James Bach, [The Nature of Exploratory Testing](https://kaner.com/pdfs/ETatQAI.pdf); James Bach, [Exploratory Testing Explained](https://www.satisfice.com/download/exploratory-testing-explained). - -## Property-based testing - -**Question:** Does an invariant hold across a generated range of inputs rather than only selected examples? - -**Relationship to TDD:** Properties can follow examples or drive a cycle when a trustworthy invariant is known. Generator and oracle quality still determine the evidence. - -**Investigate when:** Parsers, transformations, algebraic rules, protocols, or state machines have large input spaces and useful invariants. - -**Start with:** Koen Claessen and John Hughes, [QuickCheck](https://www.cs.tufts.edu/~nr/cs257/archive/john-hughes/quick.pdf). - -## Metamorphic testing - -**Question:** When an exact expected output is unavailable, should related executions still have a known relationship? - -**Relationship to TDD:** A metamorphic relation can serve as an oracle. It complements concrete examples when calculating one exact answer independently is difficult. - -**Investigate when:** Search, simulation, scientific computing, optimization, and ML systems have an oracle problem but support meaningful transformations or relations. - -**Start with:** Tsong Yueh Chen et al., [Metamorphic Testing: A Review of Challenges and Opportunities](https://doi.org/10.1145/3143561). - -## Model-based testing - -**Question:** Do generated action sequences and state transitions agree with an explicit behavioral model? - -**Relationship to TDD:** A model supplies an oracle across sequences that individual examples may miss. Creating and maintaining it is a separate design investment. - -**Investigate when:** Stateful workflows, protocols, devices, or user interfaces have many valid sequences and transition rules. - -**Start with:** Mark Utting, Alexander Pretschner, and Bruno Legeard, [A Taxonomy of Model-Based Testing Approaches](https://doi.org/10.1002/stvr.456). - -## Mutation testing - -**Question:** Would the retained tests detect selected plausible changes to production code? - -**Relationship to TDD:** Mutation probes the suite's ability to discriminate behavior. A mutation score is evidence to interpret, not a universal quality target. - -**Investigate when:** Coverage is high but assertions may be weak, critical rules deserve an evidence audit, or a team wants to find untested decision outcomes. - -**Start with:** Yue Jia and Mark Harman, [An Analysis and Survey of Mutation Testing](https://doi.org/10.1109/TSE.2010.62); Stryker, [Mutation testing elements](https://stryker-mutator.io/docs/mutation-testing-elements/). - -## Approval and snapshot testing - -**Question:** How can a team review and retain complex output when many small assertions would hide the meaningful whole? - -**Relationship to TDD:** An approved baseline can become an oracle for new or existing behavior. Snapshot storage is a mechanism; deliberate human review gives a change meaning. - -**Investigate when:** Reports, rendered output, serialized structures, compiler output, or legacy behavior are costly to express with focused assertions. - -**Start with:** ApprovalTests, [Approval Testing](https://approvaltestscpp.readthedocs.io/en/latest/generated_docs/ApprovalTestingConcept.html); Jest, [Snapshot Testing](https://jestjs.io/docs/snapshot-testing). - -## Static analysis, type systems, and formal methods - -**Question:** Which defects can be rejected or which critical properties can be justified without executing example-based tests? - -**Relationship to TDD:** Types and static checks shorten feedback for classes of invalid programs. Model checking and proof can establish properties that sampled executions cannot. - -**Investigate when:** Invalid states can be excluded by design, concurrency creates large state spaces, or failure impact justifies stronger assurance than examples provide. - -**Start with:** Benjamin Pierce, [Types and Programming Languages resources](https://www.cis.upenn.edu/~bcpierce/tapl/); Leslie Lamport, [The TLA+ Home Page](https://lamport.azurewebsites.net/tla/tla.html). - -## Quality attributes - -**Question:** Does the system remain acceptable under security, privacy, accessibility, performance, resilience, concurrency, compatibility, and resource constraints? - -**Relationship to TDD:** Some attributes can be driven by executable examples. Most also require threat models, workloads, fault injection, audits, specialist tools, or statistical evidence. - -**Investigate when:** The attribute affects users, legal duties, safety, cost, service objectives, or architecture. Define measurable quality scenarios rather than calling the concern “non-functional.” - -**Start with:** [OWASP ASVS](https://owasp.org/www-project-application-security-verification-standard/); [W3C WCAG](https://www.w3.org/TR/WCAG22/); Google, [Site Reliability Engineering](https://sre.google/books/). - -For privacy engineering, also see NIST, [Privacy Framework](https://www.nist.gov/privacy-framework). Select domain standards with qualified specialists where legal or safety obligations apply. - -## Monitoring, observability, and reconciliation - -**Question:** What can be learned only from the system operating with real traffic, dependencies, data, and failure modes? - -**Relationship to TDD:** Operational feedback detects outcomes pre-release checks cannot reproduce. It complements rather than excuses missing development evidence. - -**Investigate when:** Systems are distributed, depend on third parties, degrade gradually, process asynchronous work, or require confirmation of real business outcomes. - -**Start with:** Google SRE, [Monitoring Distributed Systems](https://sre.google/sre-book/monitoring-distributed-systems/); OpenTelemetry, [Observability primer](https://opentelemetry.io/docs/concepts/observability-primer/). - -## ML and LLM evaluation - -**Question:** How well does a probabilistic model behave across representative tasks, populations, adversarial inputs, and changing production conditions? - -**Relationship to TDD:** Test deterministic preprocessing, schemas, tool calls, and control flow normally. Model quality usually needs datasets, metrics, repeated trials, human judgment, and monitoring. - -**Investigate when:** Outputs vary, exact strings are not the real requirement, model or prompt changes shift behavior, or safety and fairness depend on a distribution of outcomes. - -**Start with:** NIST, [AI Risk Management Framework](https://www.nist.gov/itl/ai-risk-management-framework); NIST, [Generative AI Profile](https://doi.org/10.6028/NIST.AI.600-1). - -## Regulatory and safety structural coverage - -**Question:** Which prescribed evidence, traceability, independence, and structural coverage objectives apply to this product and assurance level? - -**Relationship to TDD:** TDD can contribute executable requirements and regression evidence. It does not replace mandated processes, reviews, traceability, or coverage analysis. - -**Investigate when:** Software affects aviation, automotive, medical, rail, industrial, financial, or other regulated and safety-critical domains. Engage assurance specialists early. - -**Start with:** FAA, [Advisory Circular 20-115D](https://www.faa.gov/documentLibrary/media/Advisory_Circular/AC_20-115D.pdf); ISO, [ISO 26262-6 software development](https://www.iso.org/standard/68388.html). - -Do not transfer a coverage target from one standard, assurance level, or jurisdiction to another. Determine the exact governing obligations for the system. - -## Continuous integration and delivery - -**Question:** Does work remain integrated, deployable, and evaluable as contributions and environments change? - -**Relationship to TDD:** CI extends the local feedback loop to the team. Delivery automation tests packaging and deployment paths that focused development checks cannot prove. - -**Investigate when:** More than one change stream exists, integration is delayed, releases are risky, or environment drift weakens confidence. - -**Start with:** Martin Fowler, [Continuous Integration](https://martinfowler.com/articles/continuousIntegration.html); Jez Humble, [Continuous Delivery](https://continuousdelivery.com/). - -## Architecture and walking skeletons - -**Question:** Which system boundaries, runtime responsibilities, and delivery paths must work together before local design choices can succeed? - -**Relationship to TDD:** Tests provide local design feedback but do not automatically grow a sound system architecture. A walking skeleton probes a provisional end-to-end structure early. - -**Investigate when:** Starting a system, crossing trust or deployment boundaries, making irreversible technology choices, or changing system-wide qualities. - -**Start with:** Freeman and Pryce, [Walking Skeleton](https://www.oreilly.com/library/view/growing-object-oriented-software/9780321574442/ch10.html). - -Also see Michael Nygard, [Documenting Architecture Decisions](https://www.cognitect.com/blog/2011/11/15/documenting-architecture-decisions). - -Use spikes, prototypes, architecture decision records, and quality scenarios when uncertainty requires them. Keep the probe provisional; it is not permission for comprehensive design up front. - -## How to use this appendix - -Return to the core TDD loop when the current question is behavior that can be expressed through a trustworthy executable example. - -Follow an adjacent practice when it answers a different material question. Combine evidence deliberately, and state what each result does and does not establish. diff --git a/setup/optional-skills/knowledge/tdd/guide/appendices/glossary.md b/setup/optional-skills/knowledge/tdd/guide/appendices/glossary.adoc similarity index 79% rename from setup/optional-skills/knowledge/tdd/guide/appendices/glossary.md rename to setup/optional-skills/knowledge/tdd/guide/appendices/glossary.adoc index 63aeb03..185920c 100644 --- a/setup/optional-skills/knowledge/tdd/guide/appendices/glossary.md +++ b/setup/optional-skills/knowledge/tdd/guide/appendices/glossary.adoc @@ -1,257 +1,271 @@ -# Glossary += Glossary This glossary defines terms as used in this documentation. Communities use some terms differently, especially “unit,” “mock,” “acceptance test,” and “contract test.” -## Acceptance test +== Acceptance test A test of a stakeholder-visible capability against agreed examples or criteria. It may cross several technical components without exercising a production deployment. -## Approval test +== Approval test A test that captures complex output, asks a person to review it, then compares later output with the approved baseline. -## ATDD +== ATDD -**Acceptance Test-Driven Development.** A collaborative practice in which acceptance examples are clarified before implementation and used to guide development. +*Acceptance Test-Driven Development.* A collaborative practice in which acceptance examples are clarified before implementation and used to guide development. -## BDD +== BDD -**Behavior-Driven Development.** Practices emphasizing behavior in domain language, outside-in discovery, concrete examples, and collaboration. Gherkin is optional. +*Behavior-Driven Development.* Practices emphasizing behavior in domain language, outside-in discovery, concrete examples, and collaboration. Gherkin is optional. -## Behavior +== Behavior An outcome a caller can observe under stated conditions. Behavior concerns what a system does at a chosen boundary rather than how its internals are arranged. -## Characterization test +== Characterization test A test that records what existing code currently does so it can be changed more safely. It does not imply that every captured behavior is desirable. -## Classicist TDD +== Classicist TDD A style that runs real owned collaborators together and uses doubles mainly at slow, nondeterministic, or external boundaries. Also called Detroit-style or sociable TDD. -## Component test +== Component test A test of a deployable or cohesive subsystem through its supported boundary, usually with external systems replaced or controlled. Its scope is broader than one focused rule. -## Composition root +== Composition root The application startup location where concrete dependencies are constructed and connected. Keeping construction there can leave behavior code easier to control and observe. -## Consumer-driven contract test +== Consumer-driven contract test A contract-testing workflow in which consumers publish the interactions they rely on and providers verify those interactions against their implementation. -## Continuous integration +== Continuous integration The practice of integrating changes frequently and verifying the combined state automatically. It extends local feedback into feedback about the shared codebase. -## Contract +== Contract An agreement about observable inputs, outputs, errors, or interactions at a boundary. A contract may be published between systems or internal to one codebase. -## Contract test +== Contract test A test of compatibility at a boundary between independently changing parties. Consumer-driven contract testing is one form; schema and protocol conformance are others. -## Coverage +== Coverage A structural measure reporting which statements, branches, conditions, or paths executed. It does not show by itself that results were asserted or requirements were complete. -## Double-loop TDD +== Double-loop TDD A development pattern in which an outer acceptance example gives a capability its destination while faster inner Red–Green–Refactor cycles develop the required parts. -## Dummy +== Dummy A test double passed to satisfy an interface but not used by the behavior under test. -## E2E test +== E2E test -An **end-to-end test** that observes a deployed or fully assembled path. It can reveal wiring and configuration failures but is usually slower and harder to diagnose. +An *end-to-end test* that observes a deployed or fully assembled path. It can reveal wiring and configuration failures but is usually slower and harder to diagnose. -## Executable example +== Executable example A concrete input, action, and expected outcome represented so a tool can evaluate it repeatedly. A TDD test begins as an executable example of desired behavior. -## Fake +== Fake A working implementation used in testing that takes shortcuts unsuitable for production, such as an in-memory repository with simplified semantics. -## Fake It +== Fake It A TDD strategy that uses a narrow result when the general implementation is unclear. Later examples can require generalization; refactoring may only change structure. -## Fast feedback lane +== Fast feedback lane The smallest project-defined checks used during Red, Green, and refactoring. Membership depends on measured feedback value, not on a test-level label. -## Feedback lane +== Feedback lane A project-defined group of checks selected by cost, reliability, environment, and risk. This documentation uses fast/local, standard/change-validation, and thorough/system-validation lanes. -## Feedback time +== Feedback time The time from making a change to receiving a trustworthy diagnosis that can guide the next decision. It includes execution, setup, queuing, and investigation. -## Flaky test +== Flaky test A test that can pass and fail against the same relevant code because of uncontrolled state, timing, environment, concurrency, infrastructure, or nondeterminism. -## Focused test +== Focused test A test at a narrow coherent behavior boundary, usually selected for diagnostic feedback. It may involve several functions or classes and is not guaranteed to be fast. -## Gherkin +== Gherkin A structured scenario language commonly using `Feature`, `Scenario`, `Given`, `When`, and `Then`. Using Gherkin does not by itself constitute BDD. -## Given–When–Then +== Given–When–Then A way to structure an example as relevant starting conditions, one triggering event, and observable outcomes. -## Golden master +== Golden master A reviewed output retained as the comparison baseline for an approval or characterization test. -## Green +== Green The phase in which production code is changed until the current example and relevant existing tests pass for the intended reason. -## Green integrity +== Green integrity The rule that Green is reached through a valid production change, not by weakening the oracle, skipping the test, swallowing failures, or silently changing the contract. -## Integration test +== Integration test A test that exercises real collaboration between components or infrastructure, such as application code with the production database engine. -## Mock +== Mock A test double with interaction expectations. Terminology varies by framework; this documentation distinguishes mocks from stubs and spies by built-in expectations. -## Mockist TDD +== Mockist TDD A style that uses interaction expectations to discover collaborator roles from the outside in. Also called London-style or solitary TDD. -## Mutation testing +== Mutation testing A technique that changes production code in selected ways and runs the suite to see whether tests detect those changes. -## Observation boundary +== Observation boundary The interface through which a test supplies inputs and observes outcomes. It may be a function, component, API, message contract, command, or user interface. -## Obvious Implementation +== Obvious Implementation A TDD strategy that writes the straightforward general implementation directly when the correct change is already clear. -## Oracle +== Oracle The source of truth used to decide whether an outcome is acceptable. Examples include agreed literals, domain examples, invariants, trusted references, and reviewed baselines. -## Property-based testing +== Property-based testing A technique that generates inputs to check general invariants and often shrinks a failure to a simpler counterexample. -## Red +== Red The phase in which an executable example is observed failing because the intended behavior is absent or incorrect. This documentation treats compile and missing-symbol failures as intermediate Red states. When practical, continue until the behavioral expectation itself fails. -## Red–Green–Refactor +== Red–Green–Refactor The recurring cycle: demonstrate one missing behavior, make it work, then improve structure without changing the demonstrated behavior. -## Refactor +== Refactor Change the internal structure of code without intentionally changing observable behavior. An intended contract change is not refactoring. -## Regression +== Regression A previously working and still-required behavior that stops working after a change. -## Seam +== Seam A place where behavior or a dependency can be varied without editing the code at that point. Seams can make a legacy observation boundary controllable. -## Smoke test +== Smoke test A small test of basic operability after deployment or in a production-like environment. It is intentionally shallow and fast. -## Snapshot test +== Snapshot test A baseline comparison that stores serialized or rendered output. It provides useful evidence only when changes are reviewed deliberately. -## Specification by Example +== Specification by Example A collaborative practice that discovers and documents requirements through concrete examples, often retaining them as living specifications. -## Standard feedback lane +== Standard feedback lane The project-defined checks used to validate the affected area before handoff or integration. It may contain focused, component, integration, and contract evidence. -## Spy +== Spy A test double that supplies behavior and records interactions for later assertions. -## Stub +== Stub A test double that returns controlled values or outcomes needed by the code under test. -## Succession problem +== Succession problem The problem of choosing and ordering examples so each provides useful information for the next implementation or design decision. -## TDD +== TDD -**Test-Driven Development.** A workflow that develops one observable behavior at a time through Red–Green–Refactor cycles. +*Test-Driven Development.* A workflow that develops one observable behavior at a time through Red–Green–Refactor cycles. -## Test-after +== Test-after Writing a test after the relevant production behavior exists. The test may verify and document behavior but did not influence the preceding implementation decision. -## Test double +== Test double Any substitute used in place of a real collaborator during a test. Dummies, stubs, spies, mocks, and fakes serve different purposes. -## Test-first +== Test-first Expressing intended behavior as an executable example before implementing it. Test-first enables early feedback but does not guarantee a useful test. -## Test fidelity +== Test fidelity How faithfully a check exercises the semantics relevant to the current risk. Scope and fidelity are related but separate dimensions. -## Test pyramid +== Test pyramid A portfolio heuristic favoring many narrow tests, fewer integration tests, and very few E2E tests because feedback cost often increases with scope. -## Test trophy +== Test trophy A portfolio heuristic emphasizing integration tests for applications whose main risks lie in component and framework collaboration. -## Three Amigos +== Three Amigos A collaborative example-discovery practice involving product or domain, development, and testing perspectives. -## Thorough feedback lane +== Thorough feedback lane The broadest relevant project-defined checks, often including costly or environment-dependent evidence before merge, release, or deployment. -## Triangulation +== Triangulation A TDD strategy that adds a contrasting example when one example does not justify a useful general rule. The contrast creates pressure to generalize. -## Unit test +== Unit test A test of a coherent unit of behavior at a narrow boundary. It is often fast, but “unit” does not determine cadence or mean exactly one method or class. -## Verschlimmbesserung +== Verschlimmbesserung A German term for a change intended as an improvement that makes the result worse. Retained regression tests can expose this outcome for covered behavior. -## Walking skeleton +== Walking skeleton The thinnest end-to-end implementation proving that a new system’s build, test, delivery, and runtime parts connect. It is a provisional architecture probe. + +== Sources + +Terms here are concise working definitions for this guide, not claims that all +communities use one vocabulary. See xref:../../research-findings.adoc[the +research findings] and these canonical references: + +* https://martinfowler.com/bliki/TestDrivenDevelopment.html[Test-Driven + Development] +* https://martinfowler.com/articles/mocksArentStubs.html[Mocks, stubs, and test + doubles] +* https://cucumber.io/docs/bdd/[Behavior-Driven Development] +* https://docs.pact.io/[Contract testing with Pact] +* https://glossary.istqb.org/en_US/search/test%20level[ISTQB test-level glossary] diff --git a/setup/optional-skills/knowledge/tdd/guide/appendices/stack-specific-notes.adoc b/setup/optional-skills/knowledge/tdd/guide/appendices/stack-specific-notes.adoc new file mode 100644 index 0000000..1bf12b9 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/guide/appendices/stack-specific-notes.adoc @@ -0,0 +1,55 @@ += Stack-specific notes + +These examples cover stacks used in the surrounding project. They are not an exhaustive framework catalog. + +General TDD principles remain stable while framework details change. Follow repository conventions first, then use these notes when a runtime affects test behavior or design. + +Read the core chapters before treating stack mechanics as instructions. A framework does not decide which behavior matters or which evidence answers its risk. + +== PowerShell and Pester + +* PowerShell’s success stream can turn an expected scalar into an array. Assert exact type and value when output shape is part of the contract. +* Set strict error behavior in test setup so unexpected non-terminating errors cannot masquerade as useful output. +* Pester mocks replace commands within scope and retain call history. Prefer an owned boundary when cmdlet mocking would couple behavior tests to command decomposition. +* Keep modules with business logic distinct from one-off diagnostic scripts. + +See the official https://pester.dev/docs/commands/Mock[Pester Mock command] reference. + +== Python and pytest + +* Match the existing framework. Pytest is a common default for new Python work, not a universal mandate. +* Use plain assertions and independently derived expected values. +* Default mutable fixtures to function scope. Broader fixture scope requires explicit reset or immutability. +* Patch where a name is looked up. Prefer an owned boundary when patching a third-party client would expose incidental protocol details. +* Use properties for parsers, serializers, round trips, and transformations with large input spaces. +* Test async code with the project’s async plugin and async-aware doubles. + +See the official https://docs.pytest.org/en/stable/how-to/fixtures.html[pytest fixture] and https://docs.pytest.org/en/stable/how-to/monkeypatch.html[monkeypatch] guides. + +== SQL and database-backed code + +* Use the production engine when dialect, query translation, transactions, constraints, migrations, locking, or concurrency participate in the risk. +* A fake database is appropriate only when its semantic differences cannot affect the behavior under test. +* Isolate each test’s data through rollback, schema or database isolation, or deterministic cleanup. +* Treat migrations as executable compatibility paths. Test clean installation and representative upgrades. +* Separate correctness tests from performance experiments. Performance needs representative data and a controlled environment. + +Microsoft’s https://learn.microsoft.com/en-us/ef/core/testing/choosing-a-testing-strategy[EF Core testing strategy] describes SQLite, in-memory-provider, and query differences. + +Testcontainers provides https://testcontainers.com/guides/getting-started-with-testcontainers-for-dotnet/[database testing guides] for disposable real engines. + +[[c-and-net]] +== C# and .NET + +* Follow the repository’s xUnit, NUnit, or MSTest convention and its assertion library. +* Await asynchronous operations. Blocking with `.Result` or `.Wait()` changes failure and deadlock behavior. +* Use `WebApplicationFactory` when the ASP.NET Core request pipeline is the relevant observation boundary. +* Treat EF Core’s InMemory provider and SQLite as fakes with different semantics, not proof of production-database behavior. +* Inject `TimeProvider` or another clock at time-sensitive boundaries. +* Set culture explicitly where formatting or parsing behavior depends on it. + +See Microsoft’s https://learn.microsoft.com/en-us/aspnet/core/test/integration-tests[ASP.NET Core integration-test guidance] and https://learn.microsoft.com/en-us/ef/core/testing/[EF Core testing overview]. + +== Why these notes remain an appendix + +Stack guidance is human reference, not a hard-coded skill policy. The skill inspects the repository because framework versions and project conventions change faster than the core cycle. diff --git a/setup/optional-skills/knowledge/tdd/guide/appendices/stack-specific-notes.md b/setup/optional-skills/knowledge/tdd/guide/appendices/stack-specific-notes.md deleted file mode 100644 index 6b5a61d..0000000 --- a/setup/optional-skills/knowledge/tdd/guide/appendices/stack-specific-notes.md +++ /dev/null @@ -1,54 +0,0 @@ -# Stack-specific notes - -These examples cover stacks used in the surrounding project. They are not an exhaustive framework catalog. - -General TDD principles remain stable while framework details change. Follow repository conventions first, then use these notes when a runtime affects test behavior or design. - -Read the core chapters before treating stack mechanics as instructions. A framework does not decide which behavior matters or which evidence answers its risk. - -## PowerShell and Pester - -- PowerShell’s success stream can turn an expected scalar into an array. Assert exact type and value when output shape is part of the contract. -- Set strict error behavior in test setup so unexpected non-terminating errors cannot masquerade as useful output. -- Pester mocks replace commands within scope and retain call history. Prefer an owned boundary when cmdlet mocking would couple behavior tests to command decomposition. -- Keep modules with business logic distinct from one-off diagnostic scripts. - -See the official [Pester Mock command](https://pester.dev/docs/commands/Mock) reference. - -## Python and pytest - -- Match the existing framework. Pytest is a common default for new Python work, not a universal mandate. -- Use plain assertions and independently derived expected values. -- Default mutable fixtures to function scope. Broader fixture scope requires explicit reset or immutability. -- Patch where a name is looked up. Prefer an owned boundary when patching a third-party client would expose incidental protocol details. -- Use properties for parsers, serializers, round trips, and transformations with large input spaces. -- Test async code with the project’s async plugin and async-aware doubles. - -See the official [pytest fixture](https://docs.pytest.org/en/stable/how-to/fixtures.html) and [monkeypatch](https://docs.pytest.org/en/stable/how-to/monkeypatch.html) guides. - -## SQL and database-backed code - -- Use the production engine when dialect, query translation, transactions, constraints, migrations, locking, or concurrency participate in the risk. -- A fake database is appropriate only when its semantic differences cannot affect the behavior under test. -- Isolate each test’s data through rollback, schema or database isolation, or deterministic cleanup. -- Treat migrations as executable compatibility paths. Test clean installation and representative upgrades. -- Separate correctness tests from performance experiments. Performance needs representative data and a controlled environment. - -Microsoft’s [EF Core testing strategy](https://learn.microsoft.com/en-us/ef/core/testing/choosing-a-testing-strategy) describes SQLite, in-memory-provider, and query differences. - -Testcontainers provides [database testing guides](https://testcontainers.com/guides/getting-started-with-testcontainers-for-dotnet/) for disposable real engines. - -## C# and .NET - -- Follow the repository’s xUnit, NUnit, or MSTest convention and its assertion library. -- Await asynchronous operations. Blocking with `.Result` or `.Wait()` changes failure and deadlock behavior. -- Use `WebApplicationFactory` when the ASP.NET Core request pipeline is the relevant observation boundary. -- Treat EF Core’s InMemory provider and SQLite as fakes with different semantics, not proof of production-database behavior. -- Inject `TimeProvider` or another clock at time-sensitive boundaries. -- Set culture explicitly where formatting or parsing behavior depends on it. - -See Microsoft’s [ASP.NET Core integration-test guidance](https://learn.microsoft.com/en-us/aspnet/core/test/integration-tests) and [EF Core testing overview](https://learn.microsoft.com/en-us/ef/core/testing/). - -## Why these notes remain an appendix - -Stack guidance is human reference, not a hard-coded skill policy. The skill inspects the repository because framework versions and project conventions change faster than the core cycle. diff --git a/setup/optional-skills/knowledge/tdd/research-findings.adoc b/setup/optional-skills/knowledge/tdd/research-findings.adoc new file mode 100644 index 0000000..698897c --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/research-findings.adoc @@ -0,0 +1,171 @@ += Technical Review of the TDD Foundation + +Date: 2026-07-13 +The original pre-rewrite AsciiDoc draft is not bundled because it is +superseded. Its claim disposition and useful-source mapping are preserved in +xref:analysis.adoc[the editorial assessment]. + +== Executive Assessment + +The document presents a strong, largely internally consistent *classicist TDD position*. Its core claims about short feedback loops, refactoring, the limited meaning of coverage, BDD collaboration, and complementary testing techniques are fundamentally sound. However, it is not yet a neutral “foundation”: definitions, rules of experience, and author preferences are repeatedly presented as universally applicable rules. Some claims are factually incorrect or too absolute; the most significant problems concern BDD/ATDD, the meaning of “Red,” test levels, mocks, and permanently red acceptance tests. + +== High-Priority Corrections + +[[1-red-is-not-universally-limited-to-assertion-failures]] +=== 1. “Red” is not universally limited to assertion failures + +The sections _What Counts as Red?_ and the AI rule “Red means assertion failure, not compilation failure” present a disputed preference as a definition. Robert C. Martin’s second “Law of TDD” explicitly treats a test that fails *or fails to compile* as a sufficient Red state. Martin also distinguishes the second-by-second Three Laws nano-cycle from the minute-by-minute Red–Green–Refactor cycle. The document may choose assertion-only Red as its own stricter operating rule, but it must label it accordingly and must not claim that compile-time Red is not TDD. + +Source: https://blog.cleancoder.com/uncle-bob/2014/12/17/TheCyclesOfTDD.html[Robert C. Martin, _The Cycles of TDD_] + +[[2-bdd-is-now-more-than-tdd-with-different-vocabulary]] +=== 2. BDD is now more than “TDD with different vocabulary” + +Historically, BDD emerged from Dan North’s difficulties teaching TDD; the original questions included where to start, what to test, how much to test at once, and how to name tests. North’s origin account does not support the claim that BDD was created specifically because TDD was equated with unit testing or because of coverage chasing. Current BDD practice includes Discovery, Formulation, and Automation, together with collaboratively developed shared domain understanding. Therefore, “BDD is therefore not a separate practice from TDD” and “a scenario written by one person ... is structurally indistinguishable from a well-named TDD test” are too reductive. + +Sources: https://dannorth.net/blog/introducing-bdd/[Dan North, _Introducing BDD_], https://dannorth.net/blog/bdd-is-like-tdd-if/[Dan North, _BDD is like TDD if..._], https://cucumber.io/docs/bdd/[Cucumber, _Behaviour-Driven Development_], https://agilealliance.org/glossary/bdd/[Agile Alliance, _BDD_] + +[[3-bdd-and-atdd-overlap-but-are-not-simply-synonymous]] +=== 3. BDD and ATDD overlap but are not simply synonymous + +“ATDD is, in practice, the same activity as BDD” erases a useful distinction. ATDD refers to acceptance tests created collaboratively *before implementation* from customer, development, and testing perspectives. BDD synthesizes and extends practices from TDD and ATDD through outside-in thinking, business value, shared language, and application across several abstraction levels. Some communities use the terms interchangeably; that supports describing “strong overlap,” not identity. + +Sources: https://agilealliance.org/glossary/atdd/[Agile Alliance, _ATDD_], https://agilealliance.org/glossary/bdd/[Agile Alliance, _BDD_] + +[[4-the-document-contradicts-itself-about-tdd-and-test-levels]] +=== 4. The document contradicts itself about TDD and test levels + +Early on, it correctly states that TDD is a workflow that can drive unit, integration, acceptance, and E2E tests. Later, it states that “TDD's primary domain is unit-level testing,” says TDD/BDD do not answer whether the integrated system works, and equates TDD with the unit level and BDD with the acceptance level in the comparison table. These are different models. The common definition usually describes TDD as tightly interwoven programming, *unit testing*, and refactoring; outside-in and acceptance-test-driven variants also exist. Recommendation: declare the chosen conceptual framework instead of equating a method with a level. + +Sources: https://martinfowler.com/bliki/TestDrivenDevelopment.html[Martin Fowler, _Test-Driven Development_], https://agilealliance.org/glossary/tdd/[Agile Alliance, _TDD_], https://cucumber.io/docs/bdd/[Cucumber, _BDD – Automation_] + +[[5-permanently-red-acceptance-tests-do-not-belong-in-a-normally-evaluated-ci-suite]] +=== 5. Permanently red acceptance tests do not belong in a normally evaluated CI suite + +The recommendation to add unfinished acceptance tests “red from day one” to a slow suite in which red tests are expected destroys that suite’s signal. An intentionally red test may serve as a local target or an explicitly quarantined/pending test; when run, a CI suite should be reliably green or technically distinguish a known pending state. Feature branches, tags, or disabled pending scenarios are more appropriate. Likewise, “days or weeks” as a normal Red–Green cycle is not a general TDD rule and weakens the rapid feedback emphasized by primary descriptions. + +Sources: https://martinfowler.com/bliki/TestDrivenDevelopment.html[Martin Fowler, _Test-Driven Development_], https://cucumber.io/docs/bdd/[Cucumber, _Automation_] + +[[6-mock-tests-do-not-prove-a-real-external-effect]] +=== 6. Mock tests do not prove a real external effect + +“If the interaction is the thing the user cares about (a payment was actually charged, an email was actually sent), use a mock” is inaccurately worded. A mock can prove that the code sent the expected request to a controlled abstraction; it specifically does *not* prove that the payment provider made the charge or the email was delivered. That requires contract, integration, sandbox, or E2E verification. In addition, a mock in Meszaros’ taxonomy is not simply “a spy with pre-programmed expectations,” but a double used for behavior verification; spies and mocks are related but distinct patterns. + +Sources: https://martinfowler.com/articles/mocksArentStubs.html[Martin Fowler, _Mocks Aren't Stubs_], https://xunitpatterns.com/[Gerard Meszaros, _xUnit Test Patterns_], https://docs.pact.io/[Pact, _What is contract testing?_] + +[[7-interface-changes-can-be-refactorings]] +=== 7. Interface changes can be refactorings + +The rule “Any change to that [external] interface would belong in a new red step” is too absolute. An interface change is a refactoring when all controlled callers are changed with it and the system’s observable behavior is preserved. This is considerably riskier for published APIs, but it is not excluded by definition. + +Source: https://martinfowler.com/bliki/IsChangingInterfacesRefactoring.html[Martin Fowler, _Is Changing Interfaces Refactoring?_] + +[[8-coverage-is-neither-the-goal-nor-merely-a-worthless-by-product]] +=== 8. Coverage is neither the goal nor merely a worthless by-product + +The criticism of percentage chasing is correct: high line coverage guarantees neither strong oracles nor relevant cases. However, “do not propose coverage percentage targets” and the portrayal of coverage as merely a by-product are too absolute. Coverage is a useful *gap and risk signal*; low coverage reveals unexecuted areas, and meaningful thresholds depend on criticality, change frequency, expected lifetime, and domain. The claim that TDD produces high coverage “by definition” applies only to the path currently driven by a test; refactoring can introduce new branches, and a project may use TDD only partially. In safety-critical or regulated contexts, structural coverage objectives may also be normatively required. + +Source: https://testing.googleblog.com/2020/08/code-coverage-best-practices.html[Google Testing Blog, _Code Coverage Best Practices_] + +There is also a minor error in the example: a test without an explicit assertion does fail if `calculate_total` raises an exception. It implicitly verifies “does not throw,” but not the return value. + +The AI rule that generally rejects tests without assertions or tests that only verify the absence of exceptions is also too absolute: “this operation does not throw for valid inputs” can itself be the required observable contract. The correct rule is that every test needs an effective oracle for *the behavior it claims to verify*. + +== Clarifications for Complementary Methods + +* *Mutation Testing:* The basic description is correct. “A much more honest quality measure” and direct comparisons between arbitrary mutation scores are too strong: score and cost depend on mutation operators, equivalent mutants, timeout/error classification, and the code under analysis. Even Stryker cannot reliably eliminate equivalent mutants automatically; 100% is not a meaningful universal target. Sources: https://stryker-mutator.io/docs/mutation-testing-elements/equivalent-mutants/[Stryker, _Equivalent mutants_], https://stryker-mutator.io/docs/mutation-testing-elements/mutant-states-and-metrics/[Stryker, _Mutant states and metrics_]. +* *Property-based Testing:* The definition and typical use cases are correct. “Do not suggest for ... UI” and the exclusion of glue code are acceptable heuristics but not prohibitions: stateful models can verify event sequences, protocols, and UI states. PBT does not replace examples; generator and oracle quality remain decisive. Source: https://www.cs.tufts.edu/~nr/cs257/archive/john-hughes/quick.pdf[Claessen/Hughes, _QuickCheck: A Lightweight Tool for Random Testing of Haskell Programs_]. +* *Contract Testing:* The Pact description is correct for consumer-driven contract testing, but that is not the only form of contract testing. “At every inter-service boundary” is too absolute; the technique is especially valuable for independently deployed consumer/provider teams and systems with many integrations. Contract tests verify message compatibility, not the provider’s business correctness. Source: https://docs.pact.io/[Pact, _Introduction_]. +* *Approval, Golden Master, Characterization, Snapshot:* These techniques overlap but are not stable synonyms. “Approval” describes a human-approved oracle/baseline workflow; “characterization” describes the purpose of preserving current legacy behavior; “snapshot” usually describes the stored comparison form. An approval test can verify new behavior, and a characterization test can use ordinary assertions. Sources: https://approvaltestscpp.readthedocs.io/en/latest/generated_docs/ApprovalTestingConcept.html[ApprovalTests, _ApprovalTesting concept_], https://jestjs.io/docs/snapshot-testing[Jest, _Snapshot Testing_]. +* *Testing Pyramid/Trophy:* Cohn popularized the pyramid; it is a cost heuristic, not a universal test-level standard. The claim that “each level is about an order of magnitude slower and more fragile” is unsupported and should be removed or clearly labeled a local rule of thumb. Dodds’ Trophy argues for ROI and tests that resemble real use, especially in the JavaScript/UI context; “integration tests should numerically outnumber unit tests” is one possible interpretation, not a general definition. Sources: https://martinfowler.com/bliki/TestPyramid.html[Martin Fowler, _Test Pyramid_], https://kentcdodds.com/blog/the-testing-trophy-and-testing-classifications[Kent C. Dodds, _The Testing Trophy and Testing Classifications_]. +* *Given–When–Then:* The context/event/outcome description is correct. “Exactly one `When`; split when it contains `and`” is a useful readability rule, but neither Gherkin syntax nor a universal BDD definition. Cucumber permits multiple steps and keyword sequences. Source: https://cucumber.io/docs/gherkin/reference/[Cucumber, _Gherkin Reference_]. +* *Test Levels:* “Unit / Integration / E2E / Smoke” is useful but incomplete, and definitions vary between organizations. ISO/IEC/IEEE 29119 distinguishes test processes, test levels, and test types and includes system, acceptance, performance, usability, and reliability testing. Source: https://committee.iso.org/sites/jtc1sc7/home/projects/flagship-standards/isoiecieee-29119-series.html[ISO/IEC/IEEE 29119 series overview]. + +== Empirical Evidence + +The cautious core claim “do not sell it as empirically proven” is correct. It should not, however, slide into “the evidence supports no benefit”: + +* Four industrial case studies at Microsoft/IBM reported 40–90% lower pre-release defect density with a subjectively assessed 15–35% increase in initial development time. This is positive but observational and confounded evidence, not a general causal proof. Source: https://research.ibm.com/publications/realizing-quality-improvement-through-test-driven-development-results-and-experiences-of-four-industrial-teams[Nagappan et al., 2008, IBM Research]. +* A process dissection found that small, steady steps and test granularity may explain outcomes better than test-first ordering alone. Source: https://arxiv.org/abs/1611.05994[Fucci et al., _A Dissection of the TDD Process_]. +* A larger family of experiments with TDD novices found slightly higher quality under iterative test-last than under TDD and points to training and study-duration effects. Source: https://arxiv.org/abs/2011.11942[Santos et al., _A Family of Experiments on TDD_]. + +A defensible formulation is: *The empirical results are mixed and strongly dependent on context, skill, and operationalization. There are positive and negative findings; neither side justifies universal productivity or quality promises.* The “mechanistic” benefits are also plausible mechanisms, not guaranteed outcomes: a test written first can cement a poor interface, and a test safety net makes refactoring less risky, not automatically safe. + +== Missing or Underrepresented Perspectives + +A foundation should at least mention and clearly distinguish: + +[arabic] +. *Inside-out vs. outside-in TDD* and *classicist/sociable vs. mockist/solitary TDD*. The document mentions London/classicist styles, but partly characterizes the London style inconsistently (“never mock what you own; mock only external dependencies” is not the same as “mock all interesting collaborators”). Source: https://martinfowler.com/articles/mocksArentStubs.html[Fowler, _Mocks Aren't Stubs_]. +. *Specification by Example and Example Mapping* as collaborative requirements practices; BDD is not merely a test-writing style. +. *Exploratory Testing and human/oracle questions*. Automated checks answer only explicitly encoded expectations; learning, unexpected risks, usability, and visual quality require human exploration. +. *Risk-based Testing:* Neither “every function” nor “every transformation” is the right unit. Priority follows impact, probability, change frequency, complexity, and observability. Data transformations are often good candidates, but they are not a definitional TDD boundary. +. *Non-functional quality attributes:* Security, performance, reliability/resilience, accessibility, compatibility, data quality, and observability require their own test designs and do not fit neatly into Unit/Integration/E2E. +. *Static and formal techniques:* Type checking, linting, reviews, static analysis, model checking, and formal verification complement dynamic tests; the “foundation” focuses almost entirely on executable examples. +. *Regulated/safety-critical development:* Coverage can be a required structural criterion in such contexts; “never use coverage targets” is therefore unsuitable as a universal AI rule. +. *Test maintenance and economics:* Behavior-focused tests can also break under legitimate requirement changes and create maintenance costs. “Tests should not break when implementation changes” applies only when the observed interface and semantics remain unchanged. +. *Fuzzing, metamorphic, and statistical testing:* TDD is not limited to fully deterministic logic. Randomness and concurrency can be tested through controlled seeds/seams and properties; ML/LLM behavior requires datasets, distributions, thresholds, and repeated evaluations rather than isolated deterministic assertions. +. *Flaky-test management:* Quarantine, reproducibility, ownership, and repair budgets are missing even though unstable tests substantially damage the feedback signal, especially at integration/E2E levels. + +== Claims That Can Be Retained + +* The commonly accepted core of TDD is a small test/code/refactoring cycle; refactoring is essential. https://martinfowler.com/bliki/TestDrivenDevelopment.html[Fowler] +* Test-first directs attention to the interface early; self-testing code and interface feedback are both important benefit mechanisms. https://martinfowler.com/bliki/TestDrivenDevelopment.html[Fowler] +* High line coverage guarantees neither relevant paths nor effective assertions; coverage should be used contextually as a signal. https://testing.googleblog.com/2020/08/code-coverage-best-practices.html[Google Testing Blog] +* BDD is not synonymous with Cucumber/Gherkin; collaboration and concrete examples are central. https://cucumber.io/docs/bdd/[Cucumber] +* The pyramid and trophy are context-dependent heuristics, not competing laws of nature. https://martinfowler.com/articles/2021-test-shapes.html[Fowler] +* Mockist and classicist TDD have real, documented trade-offs; a deliberate choice is better than unmarked mixing. https://martinfowler.com/articles/mocksArentStubs.html[Fowler] + +== Implications for the AI-Condensed Version + +The AI rules should use four labels: + +* *Definition:* A broadly accepted core that is non-negotiable within the term. +* *Default:* A preferred working method that may be overridden by stronger contextual signals. +* *Heuristic:* A diagnostic prompt that helps choose an action but is not a compliance rule. +* *Project Decision:* For example, assertion-only Red, classicist default, coverage gates, real database vs. fake, or testing pyramid vs. trophy. + +Absolute terms such as “never,” “always,” “at every boundary,” and “does not apply” should remain only when they protect technical correctness or safety. Most current absolutes are valuable defaults, not universal facts. + +== Propagation Status + +The second iteration applies the research through a claim-to-deliverable matrix in `traceability.adoc`. The important changes are no longer confined to this report: + +* disputed claims about Red are labeled as conventions rather than definitions; +* BDD, ATDD, Specification by Example, and double-loop TDD remain overlapping but distinct; +* acceptance examples cannot silently keep an evaluated CI suite red; +* refactoring is governed by observable behavior, not an absolute ban on interface changes; +* mocks verify requested interactions, while real effects require broader evidence; +* coverage is retained as a diagnostic and possible regulatory objective, not a quality verdict; +* classicist and London/mockist schools are defined consistently; +* companion techniques and non-functional qualities are selected from their assurance question; +* AI rules are labeled Definition, Default, Heuristic, or Project decision. + +The human chapters cite primary or authoritative sources near historical and empirical claims. The self-contained skill stores its operating reference under `develop-with-tdd/references/`. + +== Public Skill Survey + +The separate `external-skill-survey.adoc` compares public TDD, testing-strategy, contract-testing, and E2E skills through their source repositories and skills.sh entries. + +No surveyed skill supplied a stronger general TDD foundation. The useful additions were practical: contrastive examples, mutation-sensitive data, factories, provider-state contract lifecycle, E2E diagnostics, flake ownership, and gate placement. + +The local skill adopts those ideas through conditional references. It rejects fixed pyramid ratios, universal mocking rules, mandatory coverage or mutation targets, one-assertion rigidity, and framework-specific recipes in the core workflow. + +== Targeted Follow-up: Feedback, Beginners, Criticism, Greenfield, and Legacy Work + +Date: 2026-07-14 +Scope: the current `guide/01`–`07` chapters and all earlier findings in this file. + +The foundation is already strong on all five topics. It needs two concrete additions and three small emphases, not another broad chapter. + +[cols=",",options="header",] +|=== +|Topic |Finding and disposition +|*Fast feedback* |*Emphasize the mechanism.* Fast feedback is produced by one small item in flight, a prompt relevant test run, a trustworthy result, and a design signal while the decision is still cheap—not by “unit” scope or mocking alone. Beck describes seconds-scale logic and design feedback; a study of 39 professionals found fine-grained, uniform steps positively associated with quality and productivity, while test-first sequencing had no important influence. Keep “seconds” as an aspiration, not a fixed SLA, and retain the current rule that the narrowest _relevant_ boundary wins. Sources: https://newsletter.kentbeck.com/p/tdd-is-kanban-for-code[Beck, _TDD is Kanban for Code_]; https://doi.org/10.1109/TSE.2016.2616877[Fucci et al., _A Dissection of the TDD Process_]. +|*Beginner practice* |*Add a short guided practice box.* Make the test list explicit: list behavioral variants, turn exactly one item into a concrete runnable test, complete its cycle, then revise and choose again. This fills a real gap between the current compact loop and worked examples without concretizing a whole speculative suite. A controlled experiment with 48 graduate novices found better functional quality with finer-grained task descriptions; a much smaller expert/novice comparison found experts used shorter, less variable cycles. These studies justify scaffolding and coaching, not a productivity promise. Sources: https://newsletter.kentbeck.com/p/canon-tdd[Beck, _Canon TDD_]; https://doi.org/10.1109/TSE.2019.2920377[Karac, Turhan, and Juristo, novice task-granularity experiment]; https://ps.ipd.kit.edu/downloads/za_2007_effect_experience_test_driven_development.pdf[Müller and Höfer, expert/novice process study]. +|*Skeptical criticism* |*Cross-link and sharpen; do not add another debate chapter.* State once that TDD is neither a professionalism test nor inseparable from mock-heavy unit isolation. DHH’s primary objection is to test-first fundamentalism and isolation-driven indirection, while still endorsing automated regression testing; Beck likewise says there is no “gold star” for canonical conformance. Chapters 3 and 4 already answer the substantive issues by separating workflow, scope, and classicist/mockist choices. Sources: https://dhh.dk/2014/tdd-is-dead-long-live-testing.html[DHH, _TDD is dead. Long live testing._]; https://newsletter.kentbeck.com/p/canon-tdd[Beck, _Canon TDD_]. +|*Greenfield work* |*Keep the walking skeleton; add one limiting sentence.* It is a provisional end-to-end architecture probe, used to validate a broad-brush structure before focused inner cycles, not permission for comprehensive up-front architecture. The current health-path example and first stakeholder slice are otherwise sufficient. Source: https://www.oreilly.com/library/view/growing-object-oriented-software/9780321574442/ch10.html[Freeman and Pryce, __Growing Object-Oriented Software, Guided by Tests__, Chapter 10]. +|*Legacy adjustment* |*Correct the apparent ordering constraint.* Characterization and seams are already covered well, but the current numbered sequence can imply that a characterization test must always precede a seam. Sometimes a dependency prevents any test from running. Permit the smallest behavior-preserving dependency break needed to create a test point, then characterize only the behavior around the intended change, add the failing new-behavior test, and resume normal cycles. Feathers explicitly describes dependency-breaking refactorings performed without tests _in the service of putting tests in place_, and his seam chapter explains using just enough substitution to get code under test. Sources: https://www.informit.com/content/images/9780131177055/samplepages/0131177052.pdf[Feathers, _Working Effectively with Legacy Code_ sample]; https://www.informit.com/articles/article.aspx?p=359417&seqNum=2[Feathers, _The Seam Model_]. +|=== + +*Scope boundary:* Do not add rigid cycle-time limits, test-count or coverage targets, a universal unit-first/mocking rule, or causal promises from short cycles. Also exclude a kata catalog, general learning theory, IDE/framework tutorials, a full CI/CD or architecture course, and Feathers’ complete dependency-breaking catalog. Broader integration, contract, E2E, operational, and non-functional testing already has enough context in Chapter 3; expanding it further here would dilute a curriculum whose distinctive subject is the TDD feedback workflow. diff --git a/setup/optional-skills/knowledge/tdd/research-findings.md b/setup/optional-skills/knowledge/tdd/research-findings.md deleted file mode 100644 index a91089c..0000000 --- a/setup/optional-skills/knowledge/tdd/research-findings.md +++ /dev/null @@ -1,164 +0,0 @@ -# Technical Review of the TDD Foundation - -Date: 2026-07-13 -Reviewed file: `tools/tdd/notNededAnymore/testing-methodologies-foundation.adoc` - -## Executive Assessment - -The document presents a strong, largely internally consistent **classicist TDD position**. Its core claims about short feedback loops, refactoring, the limited meaning of coverage, BDD collaboration, and complementary testing techniques are fundamentally sound. However, it is not yet a neutral “foundation”: definitions, rules of experience, and author preferences are repeatedly presented as universally applicable rules. Some claims are factually incorrect or too absolute; the most significant problems concern BDD/ATDD, the meaning of “Red,” test levels, mocks, and permanently red acceptance tests. - -## High-Priority Corrections - -### 1. “Red” is not universally limited to assertion failures - -The sections *What Counts as Red?* and the AI rule “Red means assertion failure, not compilation failure” present a disputed preference as a definition. Robert C. Martin’s second “Law of TDD” explicitly treats a test that fails **or fails to compile** as a sufficient Red state. Martin also distinguishes the second-by-second Three Laws nano-cycle from the minute-by-minute Red–Green–Refactor cycle. The document may choose assertion-only Red as its own stricter operating rule, but it must label it accordingly and must not claim that compile-time Red is not TDD. - -Source: [Robert C. Martin, *The Cycles of TDD*](https://blog.cleancoder.com/uncle-bob/2014/12/17/TheCyclesOfTDD.html) - -### 2. BDD is now more than “TDD with different vocabulary” - -Historically, BDD emerged from Dan North’s difficulties teaching TDD; the original questions included where to start, what to test, how much to test at once, and how to name tests. North’s origin account does not support the claim that BDD was created specifically because TDD was equated with unit testing or because of coverage chasing. Current BDD practice includes Discovery, Formulation, and Automation, together with collaboratively developed shared domain understanding. Therefore, “BDD is therefore not a separate practice from TDD” and “a scenario written by one person ... is structurally indistinguishable from a well-named TDD test” are too reductive. - -Sources: [Dan North, *Introducing BDD*](https://dannorth.net/blog/introducing-bdd/), [Dan North, *BDD is like TDD if...*](https://dannorth.net/blog/bdd-is-like-tdd-if/), [Cucumber, *Behaviour-Driven Development*](https://cucumber.io/docs/bdd/), [Agile Alliance, *BDD*](https://agilealliance.org/glossary/bdd/) - -### 3. BDD and ATDD overlap but are not simply synonymous - -“ATDD is, in practice, the same activity as BDD” erases a useful distinction. ATDD refers to acceptance tests created collaboratively **before implementation** from customer, development, and testing perspectives. BDD synthesizes and extends practices from TDD and ATDD through outside-in thinking, business value, shared language, and application across several abstraction levels. Some communities use the terms interchangeably; that supports describing “strong overlap,” not identity. - -Sources: [Agile Alliance, *ATDD*](https://agilealliance.org/glossary/atdd/), [Agile Alliance, *BDD*](https://agilealliance.org/glossary/bdd/) - -### 4. The document contradicts itself about TDD and test levels - -Early on, it correctly states that TDD is a workflow that can drive unit, integration, acceptance, and E2E tests. Later, it states that “TDD's primary domain is unit-level testing,” says TDD/BDD do not answer whether the integrated system works, and equates TDD with the unit level and BDD with the acceptance level in the comparison table. These are different models. The common definition usually describes TDD as tightly interwoven programming, **unit testing**, and refactoring; outside-in and acceptance-test-driven variants also exist. Recommendation: declare the chosen conceptual framework instead of equating a method with a level. - -Sources: [Martin Fowler, *Test-Driven Development*](https://martinfowler.com/bliki/TestDrivenDevelopment.html), [Agile Alliance, *TDD*](https://agilealliance.org/glossary/tdd/), [Cucumber, *BDD – Automation*](https://cucumber.io/docs/bdd/) - -### 5. Permanently red acceptance tests do not belong in a normally evaluated CI suite - -The recommendation to add unfinished acceptance tests “red from day one” to a slow suite in which red tests are expected destroys that suite’s signal. An intentionally red test may serve as a local target or an explicitly quarantined/pending test; when run, a CI suite should be reliably green or technically distinguish a known pending state. Feature branches, tags, or disabled pending scenarios are more appropriate. Likewise, “days or weeks” as a normal Red–Green cycle is not a general TDD rule and weakens the rapid feedback emphasized by primary descriptions. - -Sources: [Martin Fowler, *Test-Driven Development*](https://martinfowler.com/bliki/TestDrivenDevelopment.html), [Cucumber, *Automation*](https://cucumber.io/docs/bdd/) - -### 6. Mock tests do not prove a real external effect - -“If the interaction is the thing the user cares about (a payment was actually charged, an email was actually sent), use a mock” is inaccurately worded. A mock can prove that the code sent the expected request to a controlled abstraction; it specifically does **not** prove that the payment provider made the charge or the email was delivered. That requires contract, integration, sandbox, or E2E verification. In addition, a mock in Meszaros’ taxonomy is not simply “a spy with pre-programmed expectations,” but a double used for behavior verification; spies and mocks are related but distinct patterns. - -Sources: [Martin Fowler, *Mocks Aren't Stubs*](https://martinfowler.com/articles/mocksArentStubs.html), [Gerard Meszaros, *xUnit Test Patterns*](https://xunitpatterns.com/), [Pact, *What is contract testing?*](https://docs.pact.io/) - -### 7. Interface changes can be refactorings - -The rule “Any change to that [external] interface would belong in a new red step” is too absolute. An interface change is a refactoring when all controlled callers are changed with it and the system’s observable behavior is preserved. This is considerably riskier for published APIs, but it is not excluded by definition. - -Source: [Martin Fowler, *Is Changing Interfaces Refactoring?*](https://martinfowler.com/bliki/IsChangingInterfacesRefactoring.html) - -### 8. Coverage is neither the goal nor merely a worthless by-product - -The criticism of percentage chasing is correct: high line coverage guarantees neither strong oracles nor relevant cases. However, “do not propose coverage percentage targets” and the portrayal of coverage as merely a by-product are too absolute. Coverage is a useful **gap and risk signal**; low coverage reveals unexecuted areas, and meaningful thresholds depend on criticality, change frequency, expected lifetime, and domain. The claim that TDD produces high coverage “by definition” applies only to the path currently driven by a test; refactoring can introduce new branches, and a project may use TDD only partially. In safety-critical or regulated contexts, structural coverage objectives may also be normatively required. - -Source: [Google Testing Blog, *Code Coverage Best Practices*](https://testing.googleblog.com/2020/08/code-coverage-best-practices.html) - -There is also a minor error in the example: a test without an explicit assertion does fail if `calculate_total` raises an exception. It implicitly verifies “does not throw,” but not the return value. - -The AI rule that generally rejects tests without assertions or tests that only verify the absence of exceptions is also too absolute: “this operation does not throw for valid inputs” can itself be the required observable contract. The correct rule is that every test needs an effective oracle for **the behavior it claims to verify**. - -## Clarifications for Complementary Methods - -- **Mutation Testing:** The basic description is correct. “A much more honest quality measure” and direct comparisons between arbitrary mutation scores are too strong: score and cost depend on mutation operators, equivalent mutants, timeout/error classification, and the code under analysis. Even Stryker cannot reliably eliminate equivalent mutants automatically; 100% is not a meaningful universal target. Sources: [Stryker, *Equivalent mutants*](https://stryker-mutator.io/docs/mutation-testing-elements/equivalent-mutants/), [Stryker, *Mutant states and metrics*](https://stryker-mutator.io/docs/mutation-testing-elements/mutant-states-and-metrics/). - -- **Property-based Testing:** The definition and typical use cases are correct. “Do not suggest for ... UI” and the exclusion of glue code are acceptable heuristics but not prohibitions: stateful models can verify event sequences, protocols, and UI states. PBT does not replace examples; generator and oracle quality remain decisive. Source: [Claessen/Hughes, *QuickCheck: A Lightweight Tool for Random Testing of Haskell Programs*](https://www.cs.tufts.edu/~nr/cs257/archive/john-hughes/quick.pdf). - -- **Contract Testing:** The Pact description is correct for consumer-driven contract testing, but that is not the only form of contract testing. “At every inter-service boundary” is too absolute; the technique is especially valuable for independently deployed consumer/provider teams and systems with many integrations. Contract tests verify message compatibility, not the provider’s business correctness. Source: [Pact, *Introduction*](https://docs.pact.io/). - -- **Approval, Golden Master, Characterization, Snapshot:** These techniques overlap but are not stable synonyms. “Approval” describes a human-approved oracle/baseline workflow; “characterization” describes the purpose of preserving current legacy behavior; “snapshot” usually describes the stored comparison form. An approval test can verify new behavior, and a characterization test can use ordinary assertions. Sources: [ApprovalTests, *ApprovalTesting concept*](https://approvaltestscpp.readthedocs.io/en/latest/generated_docs/ApprovalTestingConcept.html), [Jest, *Snapshot Testing*](https://jestjs.io/docs/snapshot-testing). - -- **Testing Pyramid/Trophy:** Cohn popularized the pyramid; it is a cost heuristic, not a universal test-level standard. The claim that “each level is about an order of magnitude slower and more fragile” is unsupported and should be removed or clearly labeled a local rule of thumb. Dodds’ Trophy argues for ROI and tests that resemble real use, especially in the JavaScript/UI context; “integration tests should numerically outnumber unit tests” is one possible interpretation, not a general definition. Sources: [Martin Fowler, *Test Pyramid*](https://martinfowler.com/bliki/TestPyramid.html), [Kent C. Dodds, *The Testing Trophy and Testing Classifications*](https://kentcdodds.com/blog/the-testing-trophy-and-testing-classifications). - -- **Given–When–Then:** The context/event/outcome description is correct. “Exactly one `When`; split when it contains `and`” is a useful readability rule, but neither Gherkin syntax nor a universal BDD definition. Cucumber permits multiple steps and keyword sequences. Source: [Cucumber, *Gherkin Reference*](https://cucumber.io/docs/gherkin/reference/). - -- **Test Levels:** “Unit / Integration / E2E / Smoke” is useful but incomplete, and definitions vary between organizations. ISO/IEC/IEEE 29119 distinguishes test processes, test levels, and test types and includes system, acceptance, performance, usability, and reliability testing. Source: [ISO/IEC/IEEE 29119 series overview](https://committee.iso.org/sites/jtc1sc7/home/projects/flagship-standards/isoiecieee-29119-series.html). - -## Empirical Evidence - -The cautious core claim “do not sell it as empirically proven” is correct. It should not, however, slide into “the evidence supports no benefit”: - -- Four industrial case studies at Microsoft/IBM reported 40–90% lower pre-release defect density with a subjectively assessed 15–35% increase in initial development time. This is positive but observational and confounded evidence, not a general causal proof. Source: [Nagappan et al., 2008, IBM Research](https://research.ibm.com/publications/realizing-quality-improvement-through-test-driven-development-results-and-experiences-of-four-industrial-teams). -- A process dissection found that small, steady steps and test granularity may explain outcomes better than test-first ordering alone. Source: [Fucci et al., *A Dissection of the TDD Process*](https://arxiv.org/abs/1611.05994). -- A larger family of experiments with TDD novices found slightly higher quality under iterative test-last than under TDD and points to training and study-duration effects. Source: [Santos et al., *A Family of Experiments on TDD*](https://arxiv.org/abs/2011.11942). - -A defensible formulation is: **The empirical results are mixed and strongly dependent on context, skill, and operationalization. There are positive and negative findings; neither side justifies universal productivity or quality promises.** The “mechanistic” benefits are also plausible mechanisms, not guaranteed outcomes: a test written first can cement a poor interface, and a test safety net makes refactoring less risky, not automatically safe. - -## Missing or Underrepresented Perspectives - -A foundation should at least mention and clearly distinguish: - -1. **Inside-out vs. outside-in TDD** and **classicist/sociable vs. mockist/solitary TDD**. The document mentions London/classicist styles, but partly characterizes the London style inconsistently (“never mock what you own; mock only external dependencies” is not the same as “mock all interesting collaborators”). Source: [Fowler, *Mocks Aren't Stubs*](https://martinfowler.com/articles/mocksArentStubs.html). -2. **Specification by Example and Example Mapping** as collaborative requirements practices; BDD is not merely a test-writing style. -3. **Exploratory Testing and human/oracle questions**. Automated checks answer only explicitly encoded expectations; learning, unexpected risks, usability, and visual quality require human exploration. -4. **Risk-based Testing:** Neither “every function” nor “every transformation” is the right unit. Priority follows impact, probability, change frequency, complexity, and observability. Data transformations are often good candidates, but they are not a definitional TDD boundary. -5. **Non-functional quality attributes:** Security, performance, reliability/resilience, accessibility, compatibility, data quality, and observability require their own test designs and do not fit neatly into Unit/Integration/E2E. -6. **Static and formal techniques:** Type checking, linting, reviews, static analysis, model checking, and formal verification complement dynamic tests; the “foundation” focuses almost entirely on executable examples. -7. **Regulated/safety-critical development:** Coverage can be a required structural criterion in such contexts; “never use coverage targets” is therefore unsuitable as a universal AI rule. -8. **Test maintenance and economics:** Behavior-focused tests can also break under legitimate requirement changes and create maintenance costs. “Tests should not break when implementation changes” applies only when the observed interface and semantics remain unchanged. -9. **Fuzzing, metamorphic, and statistical testing:** TDD is not limited to fully deterministic logic. Randomness and concurrency can be tested through controlled seeds/seams and properties; ML/LLM behavior requires datasets, distributions, thresholds, and repeated evaluations rather than isolated deterministic assertions. -10. **Flaky-test management:** Quarantine, reproducibility, ownership, and repair budgets are missing even though unstable tests substantially damage the feedback signal, especially at integration/E2E levels. - -## Claims That Can Be Retained - -- The commonly accepted core of TDD is a small test/code/refactoring cycle; refactoring is essential. [Fowler](https://martinfowler.com/bliki/TestDrivenDevelopment.html) -- Test-first directs attention to the interface early; self-testing code and interface feedback are both important benefit mechanisms. [Fowler](https://martinfowler.com/bliki/TestDrivenDevelopment.html) -- High line coverage guarantees neither relevant paths nor effective assertions; coverage should be used contextually as a signal. [Google Testing Blog](https://testing.googleblog.com/2020/08/code-coverage-best-practices.html) -- BDD is not synonymous with Cucumber/Gherkin; collaboration and concrete examples are central. [Cucumber](https://cucumber.io/docs/bdd/) -- The pyramid and trophy are context-dependent heuristics, not competing laws of nature. [Fowler](https://martinfowler.com/articles/2021-test-shapes.html) -- Mockist and classicist TDD have real, documented trade-offs; a deliberate choice is better than unmarked mixing. [Fowler](https://martinfowler.com/articles/mocksArentStubs.html) - -## Implications for the AI-Condensed Version - -The AI rules should use four labels: - -- **Definition:** A broadly accepted core that is non-negotiable within the term. -- **Default:** A preferred working method that may be overridden by stronger contextual signals. -- **Heuristic:** A diagnostic prompt that helps choose an action but is not a compliance rule. -- **Project Decision:** For example, assertion-only Red, classicist default, coverage gates, real database vs. fake, or testing pyramid vs. trophy. - -Absolute terms such as “never,” “always,” “at every boundary,” and “does not apply” should remain only when they protect technical correctness or safety. Most current absolutes are valuable defaults, not universal facts. - -## Propagation Status - -The second iteration applies the research through a claim-to-deliverable matrix in `traceability.md`. The important changes are no longer confined to this report: - -- disputed claims about Red are labeled as conventions rather than definitions; -- BDD, ATDD, Specification by Example, and double-loop TDD remain overlapping but distinct; -- acceptance examples cannot silently keep an evaluated CI suite red; -- refactoring is governed by observable behavior, not an absolute ban on interface changes; -- mocks verify requested interactions, while real effects require broader evidence; -- coverage is retained as a diagnostic and possible regulatory objective, not a quality verdict; -- classicist and London/mockist schools are defined consistently; -- companion techniques and non-functional qualities are selected from their assurance question; -- AI rules are labeled Definition, Default, Heuristic, or Project decision. - -The human chapters cite primary or authoritative sources near historical and empirical claims. The self-contained skill stores its operating reference under `develop-with-tdd/references/`. - -## Public Skill Survey - -The separate `external-skill-survey.md` compares public TDD, testing-strategy, contract-testing, and E2E skills through their source repositories and skills.sh entries. - -No surveyed skill supplied a stronger general TDD foundation. The useful additions were practical: contrastive examples, mutation-sensitive data, factories, provider-state contract lifecycle, E2E diagnostics, flake ownership, and gate placement. - -The local skill adopts those ideas through conditional references. It rejects fixed pyramid ratios, universal mocking rules, mandatory coverage or mutation targets, one-assertion rigidity, and framework-specific recipes in the core workflow. - -## Targeted Follow-up: Feedback, Beginners, Criticism, Greenfield, and Legacy Work - -Date: 2026-07-14 -Scope: the current `human/01`–`07` chapters and all earlier findings in this file. - -The foundation is already strong on all five topics. It needs two concrete additions and three small emphases, not another broad chapter. - -| Topic | Finding and disposition | -| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Fast feedback** | **Emphasize the mechanism.** Fast feedback is produced by one small item in flight, a prompt relevant test run, a trustworthy result, and a design signal while the decision is still cheap—not by “unit” scope or mocking alone. Beck describes seconds-scale logic and design feedback; a study of 39 professionals found fine-grained, uniform steps positively associated with quality and productivity, while test-first sequencing had no important influence. Keep “seconds” as an aspiration, not a fixed SLA, and retain the current rule that the narrowest *relevant* boundary wins. Sources: [Beck, *TDD is Kanban for Code*](https://newsletter.kentbeck.com/p/tdd-is-kanban-for-code); [Fucci et al., *A Dissection of the TDD Process*](https://doi.org/10.1109/TSE.2016.2616877). | -| **Beginner practice** | **Add a short guided practice box.** Make the test list explicit: list behavioral variants, turn exactly one item into a concrete runnable test, complete its cycle, then revise and choose again. This fills a real gap between the current compact loop and worked examples without concretizing a whole speculative suite. A controlled experiment with 48 graduate novices found better functional quality with finer-grained task descriptions; a much smaller expert/novice comparison found experts used shorter, less variable cycles. These studies justify scaffolding and coaching, not a productivity promise. Sources: [Beck, *Canon TDD*](https://newsletter.kentbeck.com/p/canon-tdd); [Karac, Turhan, and Juristo, novice task-granularity experiment](https://doi.org/10.1109/TSE.2019.2920377); [Müller and Höfer, expert/novice process study](https://ps.ipd.kit.edu/downloads/za_2007_effect_experience_test_driven_development.pdf). | -| **Skeptical criticism** | **Cross-link and sharpen; do not add another debate chapter.** State once that TDD is neither a professionalism test nor inseparable from mock-heavy unit isolation. DHH’s primary objection is to test-first fundamentalism and isolation-driven indirection, while still endorsing automated regression testing; Beck likewise says there is no “gold star” for canonical conformance. Chapters 3 and 4 already answer the substantive issues by separating workflow, scope, and classicist/mockist choices. Sources: [DHH, *TDD is dead. Long live testing.*](https://dhh.dk/2014/tdd-is-dead-long-live-testing.html); [Beck, *Canon TDD*](https://newsletter.kentbeck.com/p/canon-tdd). | -| **Greenfield work** | **Keep the walking skeleton; add one limiting sentence.** It is a provisional end-to-end architecture probe, used to validate a broad-brush structure before focused inner cycles, not permission for comprehensive up-front architecture. The current health-path example and first stakeholder slice are otherwise sufficient. Source: [Freeman and Pryce, *Growing Object-Oriented Software, Guided by Tests*, Chapter 10](https://www.oreilly.com/library/view/growing-object-oriented-software/9780321574442/ch10.html). | -| **Legacy adjustment** | **Correct the apparent ordering constraint.** Characterization and seams are already covered well, but the current numbered sequence can imply that a characterization test must always precede a seam. Sometimes a dependency prevents any test from running. Permit the smallest behavior-preserving dependency break needed to create a test point, then characterize only the behavior around the intended change, add the failing new-behavior test, and resume normal cycles. Feathers explicitly describes dependency-breaking refactorings performed without tests *in the service of putting tests in place*, and his seam chapter explains using just enough substitution to get code under test. Sources: [Feathers, *Working Effectively with Legacy Code* sample](https://www.informit.com/content/images/9780131177055/samplepages/0131177052.pdf); [Feathers, *The Seam Model*](https://www.informit.com/articles/article.aspx?p=359417&seqNum=2). | - -**Scope boundary:** Do not add rigid cycle-time limits, test-count or coverage targets, a universal unit-first/mocking rule, or causal promises from short cycles. Also exclude a kata catalog, general learning theory, IDE/framework tutorials, a full CI/CD or architecture course, and Feathers’ complete dependency-breaking catalog. Broader integration, contract, E2E, operational, and non-functional testing already has enough context in Chapter 3; expanding it further here would dilute a curriculum whose distinctive subject is the TDD feedback workflow. diff --git a/setup/optional-skills/knowledge/tdd/skill-evaluation.adoc b/setup/optional-skills/knowledge/tdd/skill-evaluation.adoc new file mode 100644 index 0000000..18cd320 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/skill-evaluation.adoc @@ -0,0 +1,106 @@ += `develop-with-tdd` forward evaluation + +Date: 2026-07-13 + +Fresh-context agents received only the skill path and task descriptions. They did not receive the expected answers or this analysis. No files were edited. + +== Evaluation criteria + +Each scenario was checked for: + +* behavior, observation-boundary, oracle, and level framing; +* a relevant Red signal before production change; +* minimal coherent Green behavior; +* a behavior-preserving refactor boundary; +* repository-aware verification; +* correct separation of defaults from product or project decisions; +* explicit limits on what the proposed test proves. + +== Results + +[cols=",,",options="header",] +|=== +|Scenario |Key behavior observed |Result +|Greenfield shipping boundary |Used exact decimal examples, drove below-threshold then inclusive boundary, and requested the missing paid-tier price |Pass +|Percentage-parser regression |Reproduced “did not raise,” guarded the numeric rule, and treated zero as a possibly first-run-passing boundary probe |Pass +|Untested legacy invoice function |Characterized only behavior at risk, separated tax-rule decisions from observed quirks, and deferred extraction until green |Pass +|EF Core uniqueness semantics |Rejected SQLite as the oracle for SQL Server collation and identified database constraint/concurrency decisions |Pass +|London/mockist order placement |Followed the repository’s interaction style, avoided claiming external effects occurred, and surfaced atomicity decisions |Pass +|Randomized retry jitter |Injected a narrow random seam, tested bounds, distinguished jitter algorithms, and separated bounded examples from distribution evidence |Pass +|=== + +== Findings + +The rule-strength taxonomy and self-contained operating reference were reached reliably. No scenario exposed a conflicting instruction or premature completion criterion. + +The evaluation also confirmed two useful behaviors not visible in metadata validation: + +* the skill asks for genuinely missing product decisions while recovering framework and repository facts itself; +* it states the assurance boundary of each test instead of inflating a focused check into system-level proof. + +== Limitation + +These evaluations tested decision process and proposed cycles against hypothetical tasks. They did not execute against six real repositories. Future changes to stack-sensitive branches should add artifact-based evaluations with runnable fixtures. + +== Progressive-disclosure evaluation + +A further fresh-context evaluation exercised the new test-design, test-level, and testability references. + +For checkout, the agent selected focused domain, SQL Server integration, consumer-contract, and one browser E2E example. Each had a distinct assurance boundary; the agent refused to claim settlement from a mock or contract. + +For hard-to-test service code, it moved SDK and clock construction to the composition root, exposed an outcome, kept deterministic helpers together under classicist TDD, and used role-shaped collaborators under London TDD. + +Result: pass. The agent loaded the conditional material and avoided one-interface-per-helper design damage. + +== Observation-boundary synchronization evaluation + +After synchronizing terminology with the human documentation, a fresh-context agent received a legacy `OrderService` task involving an internally constructed clock and payment SDK. + +The agent selected the public `requestPayment` operation as the observation boundary. It introduced clock and payment seams only to control the legacy dependencies, rather than treating the boundary itself as a seam. + +It also chose focused component evidence, derived the 30-minute oracle independently, and did not claim that a payment double proved settlement. + +Result: pass. The revised terminology changed the decision process as intended. + +== Invariant, entry-strategy, and feedback-lane evaluation + +Date: 2026-07-14 + +Fresh-context agents received the skill path and a self-contained hypothetical project description. They were told to inspect only the skill package, make no edits, and report execution honestly. + +Two initial prompts accidentally matched examples in the human curriculum and were discarded before evaluation. The replacement prompts used different domains and no expected decisions. + +[cols=",,",options="header",] +|=== +|Scenario |Observed behavior |Result +|Greenfield age-eligibility HTTP service |Asked for the missing request contract, selected the in-process HTTP boundary, used 17/18 to expose the inclusive rule, avoided mocks, designed measured fast/standard/thorough lanes, and reported that nothing ran |Pass +|Legacy report publisher with hard-wired SFTP |Used the existing smoke check as baseline, introduced a minimal Green-to-Green factory seam, rejected credential failure as Red, used an exploding factory plus returned outcome as the oracle, and preserved non-empty behavior |Pass +|Wallet withdrawal contract change |Preserved the EUR 100 example, moved the obsolete rejection boundary to EUR 201, added EUR 200 as the discriminating Red, treated the already-Green EUR 201 case as regression evidence, and retained assertion strength |Pass +|Dashboard visual exploration plus navigation rule |Kept subjective clarity in disposable user-evaluated spikes, refused to claim a winner without evidence, and isolated the deterministic empty-state navigation for a normal TDD cycle |Pass +|Pure PriceCalculator refactoring |Began from a Green public-boundary baseline, manufactured no Red, kept the private helper untested directly, and required fast evidence after structural steps plus broader validation before completion |Pass +|=== + +=== What the evaluation exercised + +* observed Red rather than inferred Red; +* known baselines and explicit environmental limitations; +* independent and outcome-complete oracles; +* intentional contract changes using added or revised evidence; +* seam-before-characterization preparation without manufacturing Red; +* pure-refactor and no-useful-oracle branches without manufactured Red; +* slower or broader evidence retained for its unique risk; +* feedback lanes separated from test levels; +* repository-wide lane and gate changes treated as team decisions; +* honest completion when no command was actually executed. + +=== Progressive-disclosure result + +The agents reached entry strategies for greenfield, legacy, and contract-change branches. Feedback-lane and broader-test material appeared only in the greenfield scenario where cadence and a process smoke test mattered. + +The core invariants remained visible without loading the explanatory operating reference. + +Result: pass. + +=== Remaining limitation + +These scenarios evaluated decisions and reporting against hypothetical projects. They did not execute a runnable fixture, so stack-specific command correctness remains covered only by future artifact-based evaluations. diff --git a/setup/optional-skills/knowledge/tdd/traceability.adoc b/setup/optional-skills/knowledge/tdd/traceability.adoc new file mode 100644 index 0000000..b3d92d9 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/traceability.adoc @@ -0,0 +1,57 @@ += Claim-to-deliverable traceability + +This matrix connects material claims from the original draft and later reviews +to their evidence, human explanation, and runtime instruction. The human +destinations are indexed in xref:README.adoc[the TDD knowledge-base index]; the +runtime destinations are in +link:../../skills/develop-with-tdd/SKILL.md[the develop-with-tdd skill]. + +“Research verdict” summarizes the relevant finding in +xref:research-findings.adoc[research-findings.adoc]. Human and skill +destinations must remain aligned when a claim changes. + +[cols=",,,",options="header",] +|=== +|Claim or decision |Research verdict |Human destination |AI/skill destination +|TDD is primarily a design-feedback discipline |Defensible thesis, not a universal definition |`guide/01-why-tdd.adoc` |`SKILL.md`; operating reference +|Prefer the shortest trustworthy loop faithful to the current risk |Retain; speed includes relevance, reliability, diagnosis, and decision timing |Human 01, 02, and 04 |Cycle invariants; feedback lanes +|Establish a known baseline |Retain as an attribution requirement |Human 01 and 02 |Cycle invariants; step 1 +|New or corrected behavior requires an observed Red |Retain as the local operating default; confidence is not evidence |Human 01–03 and 07 |Cycle invariants; step 3 +|Compile or harness failures are not automatically behavioral Red |Continue to the intended contract failure when practical |Human 01–03 |Step 3; test-design examples +|Green must preserve evidence sensitivity and the agreed contract |Retain as Green integrity |Human 02–03 and 07 |Cycle invariants; step 4 +|Refactoring preserves behavior and runs under Green |Retain; intentional behavior change starts another cycle |Human 01–03 and 07 |Cycle invariants; step 5 +|Coverage is not the goal |Retain with diagnostic and regulatory nuance |Human 01 and 06 |Operating reference +|BDD arose partly from TDD teaching difficulties |Retain; reject a single-cause coverage story |Human 05 |Operating reference +|BDD is merely renamed TDD |Reject as reductive |Human 05 |Operating reference +|BDD, ATDD, and Specification by Example are synonyms |Reject; preserve overlap and different emphases |Human 05 |Operating reference +|Test-first and test-after provide identical design feedback |Reject; post-hoc tests can still provide durable regression evidence |Human 01 and 03 |Test-design examples +|Every transforming function needs a test |Retain only as a candidate heuristic |Human 02, 03, and 06 |Operating reference +|Fake It may be generalized during refactoring |Reject when new inputs gain behavior; later examples justify generalization |Human 02, 03, and 07 |Step 4; operating reference +|Published and internal interfaces have identical refactoring constraints |Reject; use the governing observation boundary and compatibility contract |Human 02 and 03 |Step 5; operating reference +|An unfinished outer example may remain Red in required CI |Reject; keep it pending, branch-local, or outside required gates |Human 04–06 |Entry strategies +|A test without an explicit assertion has no value |Replace with an effective-oracle rule |Human 02 and appendix |Test-design examples +|Mutation score measures complete test quality |Treat as a fault-sensitivity proxy with limitations |Human 01, 06, and appendix |Operating reference +|Contract tests prove provider correctness |Narrow to exercised compatibility |Human 04, 07, and glossary |Test-level examples; broader operations +|Mocks prove payment or delivery occurred |Reject; mocks prove the visible requested interaction |Human 03, 04, 07, and appendix |Test-design examples +|London TDD mocks only external code |Reject; it commonly mocks owned collaborators |Adjacent-practices appendix |Design-for-testability +|A test pyramid prescribes universal counts |Replace with architecture, risk, and unique evidence questions |Human 04 and appendix |Test-level examples +|SQLite is a safe universal database substitute |Reject; decide from semantic fidelity |Stack appendix and Human 07 |Test-level examples +|Tests are the only assurance mechanism |Reject; include complementary evidence |Human 04–06 and appendix |Operating reference +|TDD applies only to deterministic systems |Replace with the useful-executable-oracle criterion |Human 02, 03, 06, and appendix |Entry strategies +|TDD is empirically proven superior |Reject universal causal guarantees |Human 01, 03, and 06 |Operating reference +|One test level should prove the whole feature |Reject; assign each boundary a distinct assurance question |Human 03, 04, and 07 |Test-level examples +|Mockability is the design objective |Reject; optimize controllability and observability while charging seams for indirection |Human 03, 07, and appendix |Design-for-testability +|Greenfield TDD begins with a complete up-front test design |Reject; use a provisional walking skeleton and one thin slice |Human 06 and 07 |Entry strategies +|Legacy characterization must always precede a seam |Reject; a minimal behavior-preserving seam may be needed before any test can run |Human 06 and 07 |Entry strategies +|Characterization test Green is equivalent to a TDD Red |Reject; characterization records current behavior before the new-behavior cycle |Human 03, 06, and 07 |Entry strategies +|Test level determines execution cadence |Reject; classify lanes from measured cost, reliability, environment, diagnosis, and risk |Human 04 and 07 |Feedback lanes +|Fast, standard, and thorough lanes need universal durations |Reject; use project-specific budgets and repository commands |Human 04 |Feedback lanes +|A fast irrelevant test is preferable to a slow faithful test |Reject |Human 01, 03, 04, and 07 |Cycle invariants; feedback lanes +|Unrun checks may be implied by a Green result |Reject; report commands, omitted lanes, and remaining uncertainty |Human 02, 04, and 07 |Cycle invariants; step 7 +|=== + +== Audit rule + +When a material claim changes, update its research verdict first, then every destination in that row. + +The change is complete only when human explanations, skill behavior, conditional references, examples, glossary terms, and this matrix agree. diff --git a/setup/optional-skills/knowledge/tdd/traceability.md b/setup/optional-skills/knowledge/tdd/traceability.md deleted file mode 100644 index 253d720..0000000 --- a/setup/optional-skills/knowledge/tdd/traceability.md +++ /dev/null @@ -1,49 +0,0 @@ -# Claim-to-deliverable traceability - -This matrix connects material claims from the original draft and later reviews to their evidence, human explanation, and runtime instruction. - -“Research verdict” summarizes the relevant finding in `research-findings.md`. Human and skill destinations must remain aligned when a claim changes. - -| Claim or decision | Research verdict | Human destination | AI/skill destination | -| ------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------- | ------------------------------ | --------------------------------------- | -| TDD is primarily a design-feedback discipline | Defensible thesis, not a universal definition | `human/01-why-tdd.md` | `SKILL.md`; operating reference | -| Prefer the shortest trustworthy loop faithful to the current risk | Retain; speed includes relevance, reliability, diagnosis, and decision timing | Human 01, 02, and 04 | Cycle invariants; feedback lanes | -| Establish a known baseline | Retain as an attribution requirement | Human 01 and 02 | Cycle invariants; step 1 | -| New or corrected behavior requires an observed Red | Retain as the local operating default; confidence is not evidence | Human 01–03 and 07 | Cycle invariants; step 3 | -| Compile or harness failures are not automatically behavioral Red | Continue to the intended contract failure when practical | Human 01–03 | Step 3; test-design examples | -| Green must preserve evidence sensitivity and the agreed contract | Retain as Green integrity | Human 02–03 and 07 | Cycle invariants; step 4 | -| Refactoring preserves behavior and runs under Green | Retain; intentional behavior change starts another cycle | Human 01–03 and 07 | Cycle invariants; step 5 | -| Coverage is not the goal | Retain with diagnostic and regulatory nuance | Human 01 and 06 | Operating reference | -| BDD arose partly from TDD teaching difficulties | Retain; reject a single-cause coverage story | Human 05 | Operating reference | -| BDD is merely renamed TDD | Reject as reductive | Human 05 | Operating reference | -| BDD, ATDD, and Specification by Example are synonyms | Reject; preserve overlap and different emphases | Human 05 | Operating reference | -| Test-first and test-after provide identical design feedback | Reject; post-hoc tests can still provide durable regression evidence | Human 01 and 03 | Test-design examples | -| Every transforming function needs a test | Retain only as a candidate heuristic | Human 02, 03, and 06 | Operating reference | -| Fake It may be generalized during refactoring | Reject when new inputs gain behavior; later examples justify generalization | Human 02, 03, and 07 | Step 4; operating reference | -| Published and internal interfaces have identical refactoring constraints | Reject; use the governing observation boundary and compatibility contract | Human 02 and 03 | Step 5; operating reference | -| An unfinished outer example may remain Red in required CI | Reject; keep it pending, branch-local, or outside required gates | Human 04–06 | Entry strategies | -| A test without an explicit assertion has no value | Replace with an effective-oracle rule | Human 02 and appendix | Test-design examples | -| Mutation score measures complete test quality | Treat as a fault-sensitivity proxy with limitations | Human 01, 06, and appendix | Operating reference | -| Contract tests prove provider correctness | Narrow to exercised compatibility | Human 04, 07, and glossary | Test-level examples; broader operations | -| Mocks prove payment or delivery occurred | Reject; mocks prove the visible requested interaction | Human 03, 04, 07, and appendix | Test-design examples | -| London TDD mocks only external code | Reject; it commonly mocks owned collaborators | Adjacent-practices appendix | Design-for-testability | -| A test pyramid prescribes universal counts | Replace with architecture, risk, and unique evidence questions | Human 04 and appendix | Test-level examples | -| SQLite is a safe universal database substitute | Reject; decide from semantic fidelity | Stack appendix and Human 07 | Test-level examples | -| Tests are the only assurance mechanism | Reject; include complementary evidence | Human 04–06 and appendix | Operating reference | -| TDD applies only to deterministic systems | Replace with the useful-executable-oracle criterion | Human 02, 03, 06, and appendix | Entry strategies | -| TDD is empirically proven superior | Reject universal causal guarantees | Human 01, 03, and 06 | Operating reference | -| One test level should prove the whole feature | Reject; assign each boundary a distinct assurance question | Human 03, 04, and 07 | Test-level examples | -| Mockability is the design objective | Reject; optimize controllability and observability while charging seams for indirection | Human 03, 07, and appendix | Design-for-testability | -| Greenfield TDD begins with a complete up-front test design | Reject; use a provisional walking skeleton and one thin slice | Human 06 and 07 | Entry strategies | -| Legacy characterization must always precede a seam | Reject; a minimal behavior-preserving seam may be needed before any test can run | Human 06 and 07 | Entry strategies | -| Characterization test Green is equivalent to a TDD Red | Reject; characterization records current behavior before the new-behavior cycle | Human 03, 06, and 07 | Entry strategies | -| Test level determines execution cadence | Reject; classify lanes from measured cost, reliability, environment, diagnosis, and risk | Human 04 and 07 | Feedback lanes | -| Fast, standard, and thorough lanes need universal durations | Reject; use project-specific budgets and repository commands | Human 04 | Feedback lanes | -| A fast irrelevant test is preferable to a slow faithful test | Reject | Human 01, 03, 04, and 07 | Cycle invariants; feedback lanes | -| Unrun checks may be implied by a Green result | Reject; report commands, omitted lanes, and remaining uncertainty | Human 02, 04, and 07 | Cycle invariants; step 7 | - -## Audit rule - -When a material claim changes, update its research verdict first, then every destination in that row. - -The change is complete only when human explanations, skill behavior, conditional references, examples, glossary terms, and this matrix agree. diff --git a/setup/tests/Complete-Setup.Tests.ps1 b/setup/tests/Complete-Setup.Tests.ps1 index f384fe9..9b2ab0f 100644 --- a/setup/tests/Complete-Setup.Tests.ps1 +++ b/setup/tests/Complete-Setup.Tests.ps1 @@ -34,6 +34,46 @@ Describe 'Get-RemovalPullRequestBody' -Tag 'Fast' { $script:Body | Should -Match 'Protect-MainBranch' $script:Body | Should -Match 'Enable-RepoSecurity' } + + It 'describes both setup removal and permanent-document cleanup' { + $script:Body | Should -Match 'permanent' + $script:Body | Should -Match 'reference' + } +} + +Describe 'Remove-TemplateOnlyDocumentation' -Tag 'Fast' { + It 'removes marked lines and blocks while preserving unrelated content' { + $root = Join-Path $TestDrive 'repo' + New-Item -ItemType Directory -Path $root | Out-Null + $doc = Join-Path $root 'AGENTS.md' + @' +# Keep +keep this +remove this +remove this hash-marked line # setup-teardown:template-only + +remove this block and its setup/local-link.md reference + +keep that +'@ | Set-Content -Path $doc + + $changed = Remove-TemplateOnlyDocumentation -RepositoryRoot $root -RelativePaths @('AGENTS.md') + + $changed | Should -Be @('AGENTS.md') + $result = Get-Content -Path $doc -Raw + $result | Should -Match 'keep this' + $result | Should -Match 'keep that' + $result | Should -Not -Match 'template-only|remove this|setup/' + } + + It 'returns no path when a document needs no cleanup' { + $root = Join-Path $TestDrive 'clean-repo' + New-Item -ItemType Directory -Path $root | Out-Null + Set-Content -Path (Join-Path $root 'README.md') -Value '# already clean' + + Remove-TemplateOnlyDocumentation -RepositoryRoot $root -RelativePaths @('README.md') | + Should -BeNullOrEmpty + } } Describe 'Invoke-SetupRemoval' -Tag 'Fast' { @@ -45,6 +85,7 @@ Describe 'Invoke-SetupRemoval' -Tag 'Fast' { It 'refuses to touch a template repo — no git mutation happens' { Mock Test-GhCli { $true } Mock gh { '{"isTemplate":true,"nameWithOwner":"DenWin/Template"}' } + Mock Test-AIMaintainerPrecondition { $true } Mock git { $global:LASTEXITCODE = 0 } Invoke-SetupRemoval 2>$null | Should -Be 1 Should -Invoke git -Exactly -Times 0 @@ -61,8 +102,32 @@ Describe 'Invoke-SetupRemoval' -Tag 'Fast' { Should -Invoke git -ParameterFilter { $args -contains 'rm' } -Exactly -Times 0 } - It 'removes setup/, pushes a branch, and opens the PR on the happy path' { + It 'stops before git mutation when the AI-maintainer identity is not safe' { + Mock Test-GhCli { $true } + Mock gh { '{"isTemplate":false,"nameWithOwner":"o/r"}' } + Mock Test-AIMaintainerPrecondition { $false } + Mock git { + $global:LASTEXITCODE = 0 + if ($args -contains 'status') { return '' } + } + + Invoke-SetupRemoval 2>$null | Should -Be 1 + + Should -Invoke git -ParameterFilter { $args -contains 'switch' } -Exactly -Times 0 + Should -Invoke git -ParameterFilter { $args -contains 'rm' } -Exactly -Times 0 + } + + It 'verifies identity, cleans permanent docs, removes setup, and opens the PR' { + $script:Trace = [System.Collections.Generic.List[string]]::new() Mock Test-GhCli { $true } + Mock Test-AIMaintainerPrecondition { + $script:Trace.Add('identity') + $true + } + Mock Remove-TemplateOnlyDocumentation { + $script:Trace.Add('docs') + @('AGENTS.md', 'README.md') + } Mock gh { $global:LASTEXITCODE = 0 if ($args -contains 'view') { return '{"isTemplate":false,"nameWithOwner":"o/r"}' } @@ -70,10 +135,21 @@ Describe 'Invoke-SetupRemoval' -Tag 'Fast' { } Mock git { $global:LASTEXITCODE = 0 + if ($args -contains 'switch') { $script:Trace.Add('switch') } if ($args -contains 'status') { return '' } } + Invoke-SetupRemoval | Should -Be 0 + + $script:Trace.IndexOf('identity') | Should -BeLessThan $script:Trace.IndexOf('switch') + $script:Trace.IndexOf('docs') | Should -BeGreaterThan $script:Trace.IndexOf('switch') + Should -Invoke Remove-TemplateOnlyDocumentation -Exactly -Times 1 Should -Invoke git -ParameterFilter { $args -contains 'rm' } -Exactly -Times 1 + Should -Invoke git -ParameterFilter { + $args[0] -eq 'add' -and + $args -contains 'AGENTS.md' -and + $args -contains 'README.md' + } -Exactly -Times 1 Should -Invoke git -ParameterFilter { $args -contains 'push' } -Exactly -Times 1 Should -Invoke gh -ParameterFilter { $args -contains 'create' } -Exactly -Times 1 } diff --git a/setup/tests/New-AIMaintainerApp.Tests.ps1 b/setup/tests/New-AIMaintainerApp.Tests.ps1 index 616ada9..af5adb4 100644 --- a/setup/tests/New-AIMaintainerApp.Tests.ps1 +++ b/setup/tests/New-AIMaintainerApp.Tests.ps1 @@ -66,6 +66,7 @@ Describe 'Save-AppCredential' -Tag 'Fast' { } It 'writes the private key next to nothing else and returns its path' { + Mock Protect-PrivateKeyFile {} $dir = Join-Path $TestDrive 'keys' $app = @{ id = 42; slug = 'my-bot'; pem = $script:FakePrivateKey } $path = Save-AppCredential -App $app -OutputDirectory $dir diff --git a/setup/tests/New-AIMaintainerToken.Tests.ps1 b/setup/tests/New-AIMaintainerToken.Tests.ps1 new file mode 100644 index 0000000..4c030ce --- /dev/null +++ b/setup/tests/New-AIMaintainerToken.Tests.ps1 @@ -0,0 +1,83 @@ +#Requires -Version 7.0 +Set-StrictMode -Version Latest + +BeforeAll { + . (Join-Path $PSScriptRoot '..' 'New-AIMaintainerToken.ps1') +} + +Describe 'ConvertTo-Base64Url' -Tag 'Fast' { + It 'uses URL-safe unpadded base64' { + ConvertTo-Base64Url -Bytes ([byte[]](251, 255, 239)) | + Should -Be '-__v' + } +} + +Describe 'Get-GitHubAppJwt' -Tag 'Fast' { + It 'creates a verifiable RS256 token with a short GitHub App lifetime' { + $rsa = [System.Security.Cryptography.RSA]::Create(2048) + try { + $keyPath = Join-Path $TestDrive 'app-key.pem' + [System.IO.File]::WriteAllText( + $keyPath, + $rsa.ExportRSAPrivateKeyPem(), + [System.Text.UTF8Encoding]::new($false) + ) + $now = [DateTimeOffset]::FromUnixTimeSeconds(2000000000) + + $jwt = Get-GitHubAppJwt -AppId 12345 -PrivateKeyPath $keyPath -Now $now + + $parts = $jwt -split '\.' + $parts.Count | Should -Be 3 + $payload = ConvertFrom-Base64Url -Value $parts[1] | + ForEach-Object { [System.Text.Encoding]::UTF8.GetString($_) } | + ConvertFrom-Json + $payload.iss | Should -Be '12345' + $payload.iat | Should -Be 1999999940 + $payload.exp | Should -Be 2000000540 + + $signature = ConvertFrom-Base64Url -Value $parts[2] + $rsa.VerifyData( + [System.Text.Encoding]::UTF8.GetBytes("$($parts[0]).$($parts[1])"), + $signature, + [System.Security.Cryptography.HashAlgorithmName]::SHA256, + [System.Security.Cryptography.RSASignaturePadding]::Pkcs1 + ) | Should -BeTrue + } + finally { + $rsa.Dispose() + } + } +} + +Describe 'Get-InstallationAccessToken' -Tag 'Fast' { + It 'requests a one-repository installation token without expanding permissions' { + Mock Get-GitHubAppJwt { 'signed-jwt' } + Mock gh { + $global:LASTEXITCODE = 0 + '{"token":"test-installation-token"}' + } + + $token = Get-InstallationAccessToken -AppId 123 -InstallationId 456 ` + -Repository 'owner/repo' -PrivateKeyPath 'unused.pem' + + $token | Should -Be 'test-installation-token' + Should -Invoke gh -Exactly -Times 1 -ParameterFilter { + $args -contains 'app/installations/456/access_tokens' -and + $args -contains 'Authorization: Bearer signed-jwt' -and + $args -contains 'repositories[]=repo' + } + } + + It 'fails when GitHub does not return a token' { + Mock Get-GitHubAppJwt { 'signed-jwt' } + Mock gh { + $global:LASTEXITCODE = 1 + '' + } + + { + Get-InstallationAccessToken -AppId 123 -InstallationId 456 ` + -Repository 'owner/repo' -PrivateKeyPath 'unused.pem' + } | Should -Throw + } +} diff --git a/setup/tests/Test-AIMaintainerIdentity.Tests.ps1 b/setup/tests/Test-AIMaintainerIdentity.Tests.ps1 index 853c413..e929f25 100644 --- a/setup/tests/Test-AIMaintainerIdentity.Tests.ps1 +++ b/setup/tests/Test-AIMaintainerIdentity.Tests.ps1 @@ -25,8 +25,24 @@ Describe 'Get-IdentityVerdict' -Tag 'Fast' { (Get-IdentityVerdict -TokenKind 'Installation' -HasAdmin $false -HasWrite $true).Pass | Should -BeTrue } - It 'passes a fine-grained PAT without admin' { - (Get-IdentityVerdict -TokenKind 'FineGrainedPat' -HasAdmin $false -HasWrite $true).Pass | Should -BeTrue + It 'passes a fine-grained PAT owned by a separate organization member' { + $v = Get-IdentityVerdict -TokenKind 'FineGrainedPat' -HasAdmin $false -HasWrite $true ` + -ActorLogin 'maintainer-bot' -RepositoryOwner 'example-org' + $v.Pass | Should -BeTrue + } + + It 'rejects a fine-grained PAT owned by the personal repository owner' { + $v = Get-IdentityVerdict -TokenKind 'FineGrainedPat' -HasAdmin $false -HasWrite $true ` + -ActorLogin 'owner' -RepositoryOwner 'owner' + $v.Pass | Should -BeFalse + $v.Findings -join ' ' | Should -Match 'owner|separate identity' + } + + It 'fails closed when a fine-grained PAT actor cannot be identified' { + $v = Get-IdentityVerdict -TokenKind 'FineGrainedPat' -HasAdmin $false -HasWrite $true ` + -RepositoryOwner 'example-org' + $v.Pass | Should -BeFalse + $v.Findings -join ' ' | Should -Match 'verify|identity' } It 'fails any credential that has admin on the repo, naming the containment risk' { @@ -91,10 +107,24 @@ Describe 'Invoke-IdentityCheck' -Tag 'Fast' { Mock gh { if ($args -contains 'view') { return 'owner/repo' } if ($args -contains 'token') { return 'github_pat_11ABC123_abcdef' } # gitleaks:allow + if (($args -join ' ') -match 'api user') { return 'maintainer-bot' } if (($args -join ' ') -match 'permissions\.admin') { $global:LASTEXITCODE = 0; return 'false' } if (($args -join ' ') -match 'permissions\.push') { $global:LASTEXITCODE = 0; return 'false' } $global:LASTEXITCODE = 0 } Invoke-IdentityCheck 2>$null | Should -Be 1 } + + It 'returns non-zero for the repository owner fine-grained PAT' { + Mock Test-GhCli { $true } + Mock gh { + if ($args -contains 'view') { return 'owner/repo' } + if ($args -contains 'token') { return 'github_pat_11ABC123_abcdef' } # gitleaks:allow + if (($args -join ' ') -match 'api user') { return 'owner' } + if (($args -join ' ') -match 'permissions\.admin') { $global:LASTEXITCODE = 0; return 'false' } + if (($args -join ' ') -match 'permissions\.push') { $global:LASTEXITCODE = 0; return 'true' } + $global:LASTEXITCODE = 0 + } + Invoke-IdentityCheck 2>$null | Should -Be 1 + } } diff --git a/setup/version b/setup/version index 0d91a54..1d0ba9e 100644 --- a/setup/version +++ b/setup/version @@ -1 +1 @@ -0.3.0 +0.4.0 From f0f723b593a8bb87bcffffd3644d9904566ebe2b Mon Sep 17 00:00:00 2001 From: Dennis Winter Date: Sat, 18 Jul 2026 18:02:46 +0200 Subject: [PATCH 8/8] Address AI-maintainer review findings - verify the requested repository installation before minting App tokens - bind setup-removal pushes to the validated credential over HTTPS - keep the temporary CODEOWNERS rule valid and teardown-safe --- .github/CODEOWNERS | 4 +- setup/AI-Maintainer-Identity.adoc | 29 +++- setup/CHANGELOG.md | 3 + setup/Complete-Setup.ps1 | 109 +++++++++++++- setup/New-AIMaintainerToken.ps1 | 16 +++ setup/tests/Complete-Setup.Tests.ps1 | 148 +++++++++++++++++++- setup/tests/New-AIMaintainerToken.Tests.ps1 | 42 +++++- 7 files changed, 341 insertions(+), 10 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e5b097f..493559f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,6 +4,8 @@ /.github/ @DenWin /.pre-commit-config.yaml @DenWin /.config/ @DenWin -/setup/ @DenWin # setup-teardown:template-only +# setup-teardown:template-only:start +/setup/ @DenWin +# setup-teardown:template-only:end /AGENTS.md @DenWin /CLAUDE.md @DenWin diff --git a/setup/AI-Maintainer-Identity.adoc b/setup/AI-Maintainer-Identity.adoc index c6b7ab5..5ddff48 100644 --- a/setup/AI-Maintainer-Identity.adoc +++ b/setup/AI-Maintainer-Identity.adoc @@ -144,13 +144,36 @@ $env:GH_TOKEN = pwsh -NoProfile -File setup/New-AIMaintainerToken.ps1 ` -Repository owner/name ` -PrivateKeyPath ~/.github-apps/my-agent.private-key.pem +$appSlug = 'my-ai-maintainer' +$botLogin = "${appSlug}[bot]" +$botId = gh api "users/$botLogin" --jq .id +git config --local user.name $botLogin +git config --local user.email "$botId+$botLogin@users.noreply.github.com" + gh auth setup-git pwsh -NoProfile -File setup/Test-AIMaintainerIdentity.ps1 +git var GIT_AUTHOR_IDENT +git var GIT_COMMITTER_IDENT ---- -Give the agent only this shell. The token is scoped to the named repository and -is separate from your personal `gh auth login` session. Mint a new one after -expiry; never persist it in the repository or shell profile. +Replace `my-ai-maintainer` with the App slug. Repository-local `user.name` and +`user.email` make local commits identify the App bot; authentication alone does +not set commit authorship. Git documents that these values populate both author +and committer fields +(https://git-scm.com/docs/git-config#Documentation/git-config.txt-username[Git +configuration]), while `gh auth setup-git` only configures a credential helper +(https://cli.github.com/manual/gh_auth_setup-git[GitHub CLI manual]). The +ID-based no-reply address follows GitHub's documented +`ID+USERNAME@users.noreply.github.com` format +(https://docs.github.com/en/account-and-profile/reference/email-addresses-reference#your-noreply-email-address[GitHub +email reference]). + +Give the agent only this shell. The token helper first verifies that the supplied +installation ID belongs to the full `owner/name`, then scopes the token to that +repository. `Complete-Setup.ps1` also forces its push through HTTPS with the +credential that passed verification, regardless of the clone's `origin` +transport. Mint a new token after expiry; never persist it in the repository or +shell profile. == Option B — Fine-grained PAT diff --git a/setup/CHANGELOG.md b/setup/CHANGELOG.md index 18c9bfe..f9959d5 100644 --- a/setup/CHANGELOG.md +++ b/setup/CHANGELOG.md @@ -37,6 +37,9 @@ records only what changed. - `Complete-Setup.ps1` now verifies the AI-maintainer identity and removes dangling `setup/` references from permanent documents. +- GitHub App tokens now verify the full repository installation before minting; + setup-removal pushes use the checked token over HTTPS, and the temporary + `/setup/` CODEOWNERS rule remains valid until teardown removes it. - Semgrep and GitHub behavior claims now link to current primary sources and no longer present historical Windows or plan constraints as universal. diff --git a/setup/Complete-Setup.ps1 b/setup/Complete-Setup.ps1 index 6c0d801..db83ba8 100644 --- a/setup/Complete-Setup.ps1 +++ b/setup/Complete-Setup.ps1 @@ -62,7 +62,8 @@ function Remove-TemplateOnlyDocumentation { ) $changed = [System.Collections.Generic.List[string]]::new() - $blockPattern = '(?ms)^[ \t]*\r?\n.*?^[ \t]*\r?\n?' + $htmlBlockPattern = '(?ms)^[ \t]*\r?\n.*?^[ \t]*\r?\n?' + $hashBlockPattern = '(?ms)^[ \t]*# setup-teardown:template-only:start[ \t]*\r?\n.*?^[ \t]*# setup-teardown:template-only:end[ \t]*(?:\r?\n|$)' $linePattern = '(?m)^.*(?:|# setup-teardown:template-only).*(?:\r?\n|$)' foreach ($relativePath in $RelativePaths) { @@ -72,7 +73,8 @@ function Remove-TemplateOnlyDocumentation { } $original = [System.IO.File]::ReadAllText($path) - $updated = [regex]::Replace($original, $blockPattern, '') + $updated = [regex]::Replace($original, $htmlBlockPattern, '') + $updated = [regex]::Replace($updated, $hashBlockPattern, '') $updated = [regex]::Replace($updated, $linePattern, '') if ($updated -ne $original -and @@ -89,6 +91,99 @@ function Remove-TemplateOnlyDocumentation { $changed } +function Invoke-AuthenticatedGitPush { + <# Force this push through GitHub HTTPS with the exact token that passed the + identity check. Process-scoped Git config avoids putting the token in + argv or changing the user's persistent credential helpers. #> + param( + [Parameter(Mandatory)] + [ValidatePattern('^[^/]+/[^/]+$')] + [string]$Repository, + [Parameter(Mandatory)] + [string]$BranchName, + [Parameter(Mandatory)] + [string]$Token + ) + + $existingCount = 0 + if (-not [string]::IsNullOrWhiteSpace($env:GIT_CONFIG_COUNT) -and + (-not [int]::TryParse($env:GIT_CONFIG_COUNT, [ref]$existingCount) -or + $existingCount -lt 0)) { + throw 'GIT_CONFIG_COUNT must be a non-negative integer.' + } + + $firstIndex = $existingCount + $managedNames = @( + 'GIT_CONFIG_COUNT', + "GIT_CONFIG_KEY_$firstIndex", + "GIT_CONFIG_VALUE_$firstIndex", + "GIT_CONFIG_KEY_$($firstIndex + 1)", + "GIT_CONFIG_VALUE_$($firstIndex + 1)", + "GIT_CONFIG_KEY_$($firstIndex + 2)", + "GIT_CONFIG_VALUE_$($firstIndex + 2)", + 'GIT_TERMINAL_PROMPT' + ) + $savedEnvironment = @{} + foreach ($name in $managedNames) { + $savedEnvironment[$name] = [Environment]::GetEnvironmentVariable($name, 'Process') + } + + $basicCredential = [Convert]::ToBase64String( + [System.Text.Encoding]::ASCII.GetBytes("x-access-token:$Token") + ) + + try { + [Environment]::SetEnvironmentVariable( + 'GIT_CONFIG_COUNT', + ($existingCount + 3).ToString(), + 'Process' + ) + [Environment]::SetEnvironmentVariable( + "GIT_CONFIG_KEY_$firstIndex", + 'http.https://github.com/.extraHeader', + 'Process' + ) + [Environment]::SetEnvironmentVariable( + "GIT_CONFIG_VALUE_$firstIndex", + '', + 'Process' + ) + [Environment]::SetEnvironmentVariable( + "GIT_CONFIG_KEY_$($firstIndex + 1)", + 'http.https://github.com/.extraHeader', + 'Process' + ) + [Environment]::SetEnvironmentVariable( + "GIT_CONFIG_VALUE_$($firstIndex + 1)", + "Authorization: Basic $basicCredential", + 'Process' + ) + [Environment]::SetEnvironmentVariable( + "GIT_CONFIG_KEY_$($firstIndex + 2)", + 'credential.helper', + 'Process' + ) + [Environment]::SetEnvironmentVariable( + "GIT_CONFIG_VALUE_$($firstIndex + 2)", + '', + 'Process' + ) + [Environment]::SetEnvironmentVariable('GIT_TERMINAL_PROMPT', '0', 'Process') + + git push -u "https://github.com/$Repository.git" $BranchName + return $LASTEXITCODE -eq 0 + } + finally { + foreach ($name in $managedNames) { + [Environment]::SetEnvironmentVariable( + $name, + $savedEnvironment[$name], + 'Process' + ) + } + } +} + function Test-RemovalAllowed { <# The template failsafe (pure; no side effects): a template repo keeps its setup/ folder — only repos CREATED from it remove theirs. #> @@ -166,6 +261,12 @@ function Invoke-SetupRemoval { return 1 } + $pushToken = gh auth token 2>$null + if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($pushToken)) { + Write-Error 'Could not read the verified GitHub credential for the setup-removal push.' + return 1 + } + git switch -c $BranchName if ($LASTEXITCODE -ne 0) { Write-Error "Could not create branch '$BranchName' (does it already exist?)." @@ -197,8 +298,8 @@ function Invoke-SetupRemoval { Write-Error 'Commit failed.' return 1 } - git push -u origin $BranchName - if ($LASTEXITCODE -ne 0) { + if (-not (Invoke-AuthenticatedGitPush -Repository $repoInfo.nameWithOwner ` + -BranchName $BranchName -Token $pushToken)) { Write-Error 'Push failed — check your remote and permissions.' return 1 } diff --git a/setup/New-AIMaintainerToken.ps1 b/setup/New-AIMaintainerToken.ps1 index aa26ac7..3400eb6 100644 --- a/setup/New-AIMaintainerToken.ps1 +++ b/setup/New-AIMaintainerToken.ps1 @@ -87,6 +87,22 @@ function Get-InstallationAccessToken { ) $jwt = Get-GitHubAppJwt -AppId $AppId -PrivateKeyPath $PrivateKeyPath + $installationResponse = gh api ` + -H 'Accept: application/vnd.github+json' ` + -H "Authorization: Bearer $jwt" ` + -H 'X-GitHub-Api-Version: 2022-11-28' ` + "repos/$Repository/installation" 2>$null + + if ($LASTEXITCODE -ne 0 -or -not $installationResponse) { + throw "GitHub App installation not found for '$Repository'. Check the App ID, private key, and repository installation." + } + + $repositoryInstallationId = ($installationResponse | ConvertFrom-Json).id + if (-not $repositoryInstallationId -or + [long]$repositoryInstallationId -ne $InstallationId) { + throw "Installation ID '$InstallationId' does not belong to repository '$Repository'." + } + $repositoryName = ($Repository -split '/', 2)[1] $response = gh api --method POST ` -H 'Accept: application/vnd.github+json' ` diff --git a/setup/tests/Complete-Setup.Tests.ps1 b/setup/tests/Complete-Setup.Tests.ps1 index 9b2ab0f..5ee59d7 100644 --- a/setup/tests/Complete-Setup.Tests.ps1 +++ b/setup/tests/Complete-Setup.Tests.ps1 @@ -74,6 +74,129 @@ keep that Remove-TemplateOnlyDocumentation -RepositoryRoot $root -RelativePaths @('README.md') | Should -BeNullOrEmpty } + + It 'removes a CODEOWNERS-safe hash-comment block including the setup rule' { + $root = Join-Path $TestDrive 'codeowners-repo' + $github = Join-Path $root '.github' + New-Item -ItemType Directory -Path $github | Out-Null + $codeowners = Join-Path $github 'CODEOWNERS' + @' +/.github/ @owner +# setup-teardown:template-only:start +/setup/ @owner +# setup-teardown:template-only:end +/AGENTS.md @owner +'@ | Set-Content -Path $codeowners + + $changed = Remove-TemplateOnlyDocumentation -RepositoryRoot $root ` + -RelativePaths @('.github/CODEOWNERS') + + $changed | Should -Be @('.github/CODEOWNERS') + $result = Get-Content -Path $codeowners -Raw + $result | Should -Match ([regex]::Escape('/.github/ @owner')) + $result | Should -Match ([regex]::Escape('/AGENTS.md @owner')) + $result | Should -Not -Match 'setup|template-only' + } + + It 'keeps the repository CODEOWNERS rule valid before teardown and removes it after' { + $source = Join-Path $PSScriptRoot '..' '..' '.github' 'CODEOWNERS' + $original = Get-Content -LiteralPath $source -Raw + $original | Should -Match '(?m)^/setup/\s+@DenWin\s*$' + $original | Should -Not -Match '(?m)^/setup/.*#' + + $root = Join-Path $TestDrive 'current-codeowners-repo' + $github = Join-Path $root '.github' + New-Item -ItemType Directory -Path $github | Out-Null + $copy = Join-Path $github 'CODEOWNERS' + Copy-Item -LiteralPath $source -Destination $copy + + Remove-TemplateOnlyDocumentation -RepositoryRoot $root ` + -RelativePaths @('.github/CODEOWNERS') | Should -Be @('.github/CODEOWNERS') + + $cleaned = Get-Content -LiteralPath $copy -Raw + $cleaned | Should -Not -Match 'setup|template-only' + $cleaned | Should -Match ([regex]::Escape('/.github/')) + $cleaned | Should -Match ([regex]::Escape('/AGENTS.md')) + } +} + +Describe 'Invoke-AuthenticatedGitPush' -Tag 'Fast' { + It 'uses the validated token for an explicit HTTPS push without putting it in argv' { + $saved = @{ + Count = $env:GIT_CONFIG_COUNT + Key0 = $env:GIT_CONFIG_KEY_0 + Value0 = $env:GIT_CONFIG_VALUE_0 + Terminal = $env:GIT_TERMINAL_PROMPT + } + try { + $env:GIT_CONFIG_COUNT = '1' + $env:GIT_CONFIG_KEY_0 = 'existing.key' + $env:GIT_CONFIG_VALUE_0 = 'existing-value' + $env:GIT_TERMINAL_PROMPT = 'existing-prompt' + $script:PushEnvironment = $null + + Mock git { + $script:PushEnvironment = @{ + Count = $env:GIT_CONFIG_COUNT + Key0 = $env:GIT_CONFIG_KEY_0 + Value0 = $env:GIT_CONFIG_VALUE_0 + Key1 = $env:GIT_CONFIG_KEY_1 + Value1 = $env:GIT_CONFIG_VALUE_1 + Key2 = $env:GIT_CONFIG_KEY_2 + Value2 = $env:GIT_CONFIG_VALUE_2 + Key3 = $env:GIT_CONFIG_KEY_3 + Value3 = $env:GIT_CONFIG_VALUE_3 + Terminal = $env:GIT_TERMINAL_PROMPT + } + $global:LASTEXITCODE = 0 + } + + Invoke-AuthenticatedGitPush -Repository 'owner/repo' ` + -BranchName 'chore/remove-setup' -Token 'test-installation-token' | + Should -BeTrue + + $expected = [Convert]::ToBase64String( + [System.Text.Encoding]::ASCII.GetBytes('x-access-token:test-installation-token') + ) + $script:PushEnvironment.Count | Should -Be '4' + $script:PushEnvironment.Key0 | Should -Be 'existing.key' + $script:PushEnvironment.Value0 | Should -Be 'existing-value' + $script:PushEnvironment.Key1 | Should -Be 'http.https://github.com/.extraHeader' + $script:PushEnvironment.Value1 | Should -Be '' + $script:PushEnvironment.Key2 | Should -Be 'http.https://github.com/.extraHeader' + $script:PushEnvironment.Value2 | Should -Be "Authorization: Basic $expected" + $script:PushEnvironment.Key3 | Should -Be 'credential.helper' + $script:PushEnvironment.Value3 | Should -Be '' + $script:PushEnvironment.Terminal | Should -Be '0' + Should -Invoke git -Exactly -Times 1 -ParameterFilter { + $args[0] -eq 'push' -and + $args[1] -eq '-u' -and + $args[2] -eq 'https://github.com/owner/repo.git' -and + $args[3] -eq 'chore/remove-setup' -and + ($args -join ' ') -notmatch 'test-installation-token' + } + + $env:GIT_CONFIG_COUNT | Should -Be '1' + $env:GIT_CONFIG_KEY_0 | Should -Be 'existing.key' + $env:GIT_CONFIG_VALUE_0 | Should -Be 'existing-value' + $env:GIT_TERMINAL_PROMPT | Should -Be 'existing-prompt' + $env:GIT_CONFIG_KEY_1 | Should -BeNullOrEmpty + $env:GIT_CONFIG_VALUE_1 | Should -BeNullOrEmpty + $env:GIT_CONFIG_KEY_2 | Should -BeNullOrEmpty + $env:GIT_CONFIG_VALUE_2 | Should -BeNullOrEmpty + $env:GIT_CONFIG_KEY_3 | Should -BeNullOrEmpty + $env:GIT_CONFIG_VALUE_3 | Should -BeNullOrEmpty + } + finally { + $env:GIT_CONFIG_COUNT = $saved.Count + $env:GIT_CONFIG_KEY_0 = $saved.Key0 + $env:GIT_CONFIG_VALUE_0 = $saved.Value0 + $env:GIT_TERMINAL_PROMPT = $saved.Terminal + Remove-Item Env:GIT_CONFIG_KEY_1, Env:GIT_CONFIG_VALUE_1, + Env:GIT_CONFIG_KEY_2, Env:GIT_CONFIG_VALUE_2, + Env:GIT_CONFIG_KEY_3, Env:GIT_CONFIG_VALUE_3 -ErrorAction SilentlyContinue + } + } } Describe 'Invoke-SetupRemoval' -Tag 'Fast' { @@ -117,6 +240,25 @@ Describe 'Invoke-SetupRemoval' -Tag 'Fast' { Should -Invoke git -ParameterFilter { $args -contains 'rm' } -Exactly -Times 0 } + It 'stops before branching when the verified push token cannot be read' { + Mock Test-GhCli { $true } + Mock Test-AIMaintainerPrecondition { $true } + Mock gh { + $global:LASTEXITCODE = 0 + if ($args -contains 'view') { return '{"isTemplate":false,"nameWithOwner":"o/r"}' } + if ($args -contains 'token') { return '' } + } + Mock git { + $global:LASTEXITCODE = 0 + if ($args -contains 'status') { return '' } + } + + Invoke-SetupRemoval 2>$null | Should -Be 1 + + Should -Invoke git -ParameterFilter { $args -contains 'switch' } -Exactly -Times 0 + Should -Invoke git -ParameterFilter { $args -contains 'push' } -Exactly -Times 0 + } + It 'verifies identity, cleans permanent docs, removes setup, and opens the PR' { $script:Trace = [System.Collections.Generic.List[string]]::new() Mock Test-GhCli { $true } @@ -131,6 +273,7 @@ Describe 'Invoke-SetupRemoval' -Tag 'Fast' { Mock gh { $global:LASTEXITCODE = 0 if ($args -contains 'view') { return '{"isTemplate":false,"nameWithOwner":"o/r"}' } + if ($args -contains 'token') { return 'validated-installation-token' } if ($args -contains 'create') { return 'https://github.com/o/r/pull/1' } } Mock git { @@ -150,7 +293,10 @@ Describe 'Invoke-SetupRemoval' -Tag 'Fast' { $args -contains 'AGENTS.md' -and $args -contains 'README.md' } -Exactly -Times 1 - Should -Invoke git -ParameterFilter { $args -contains 'push' } -Exactly -Times 1 + Should -Invoke git -ParameterFilter { + $args[0] -eq 'push' -and + $args[2] -eq 'https://github.com/o/r.git' + } -Exactly -Times 1 Should -Invoke gh -ParameterFilter { $args -contains 'create' } -Exactly -Times 1 } } diff --git a/setup/tests/New-AIMaintainerToken.Tests.ps1 b/setup/tests/New-AIMaintainerToken.Tests.ps1 index 4c030ce..0645b59 100644 --- a/setup/tests/New-AIMaintainerToken.Tests.ps1 +++ b/setup/tests/New-AIMaintainerToken.Tests.ps1 @@ -50,10 +50,13 @@ Describe 'Get-GitHubAppJwt' -Tag 'Fast' { } Describe 'Get-InstallationAccessToken' -Tag 'Fast' { - It 'requests a one-repository installation token without expanding permissions' { + It 'verifies the owner/name installation before requesting a one-repository token' { Mock Get-GitHubAppJwt { 'signed-jwt' } Mock gh { $global:LASTEXITCODE = 0 + if ($args -contains 'repos/owner/repo/installation') { + return '{"id":456}' + } '{"token":"test-installation-token"}' } @@ -61,6 +64,10 @@ Describe 'Get-InstallationAccessToken' -Tag 'Fast' { -Repository 'owner/repo' -PrivateKeyPath 'unused.pem' $token | Should -Be 'test-installation-token' + Should -Invoke gh -Exactly -Times 1 -ParameterFilter { + $args -contains 'repos/owner/repo/installation' -and + $args -contains 'Authorization: Bearer signed-jwt' + } Should -Invoke gh -Exactly -Times 1 -ParameterFilter { $args -contains 'app/installations/456/access_tokens' -and $args -contains 'Authorization: Bearer signed-jwt' -and @@ -68,6 +75,26 @@ Describe 'Get-InstallationAccessToken' -Tag 'Fast' { } } + It 'rejects an installation ID belonging to a different repository owner' { + Mock Get-GitHubAppJwt { 'signed-jwt' } + Mock gh { + $global:LASTEXITCODE = 0 + if ($args -contains 'repos/org-b/repo/installation') { + return '{"id":999}' + } + '{"token":"wrong-repository-token"}' + } + + { + Get-InstallationAccessToken -AppId 123 -InstallationId 456 ` + -Repository 'org-b/repo' -PrivateKeyPath 'unused.pem' + } | Should -Throw '*installation*' + + Should -Invoke gh -Exactly -Times 0 -ParameterFilter { + $args -contains 'app/installations/456/access_tokens' + } + } + It 'fails when GitHub does not return a token' { Mock Get-GitHubAppJwt { 'signed-jwt' } Mock gh { @@ -81,3 +108,16 @@ Describe 'Get-InstallationAccessToken' -Tag 'Fast' { } | Should -Throw } } + +Describe 'GitHub App identity instructions' -Tag 'Fast' { + It 'sets repository-local bot author and committer identity' { + $identityGuide = Get-Content -LiteralPath ( + Join-Path $PSScriptRoot '..' 'AI-Maintainer-Identity.adoc' + ) -Raw + + $identityGuide | Should -Match ([regex]::Escape('git config --local user.name')) + $identityGuide | Should -Match ([regex]::Escape('git config --local user.email')) + $identityGuide | Should -Match ([regex]::Escape('git var GIT_AUTHOR_IDENT')) + $identityGuide | Should -Match ([regex]::Escape('git var GIT_COMMITTER_IDENT')) + } +}